aboutsummaryrefslogtreecommitdiffstats
path: root/mm
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2012-12-18 13:56:07 -0500
committerLinus Torvalds <torvalds@linux-foundation.org>2012-12-18 13:56:07 -0500
commitae664dba2724e59ddd66291b895f7370e28b9a7a (patch)
treed6e214bdc9999bcb8b0a067053aa6934cfd9d60e /mm
parenta2faf2fc534f57ba26bc4d613795236ed4f5fb1c (diff)
parent08afe22c68d8c07e8e31ee6491c37f36199ba14b (diff)
Merge branch 'slab/for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/penberg/linux
Pull SLAB changes from Pekka Enberg: "This contains preparational work from Christoph Lameter and Glauber Costa for SLAB memcg and cleanups and improvements from Ezequiel Garcia and Joonsoo Kim. Please note that the SLOB cleanup commit from Arnd Bergmann already appears in your tree but I had also merged it myself which is why it shows up in the shortlog." * 'slab/for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/penberg/linux: mm/sl[aou]b: Common alignment code slab: Use the new create_boot_cache function to simplify bootstrap slub: Use statically allocated kmem_cache boot structure for bootstrap mm, sl[au]b: create common functions for boot slab creation slab: Simplify bootstrap slub: Use correct cpu_slab on dead cpu mm: fix slab.c kernel-doc warnings mm/slob: use min_t() to compare ARCH_SLAB_MINALIGN slab: Ignore internal flags in cache creation mm/slob: Use free_page instead of put_page for page-size kmalloc allocations mm/sl[aou]b: Move common kmem_cache_size() to slab.h mm/slob: Use object_size field in kmem_cache_size() mm/slob: Drop usage of page->private for storing page-sized allocations slub: Commonize slab_cache field in struct page sl[au]b: Process slabinfo_show in common code mm/sl[au]b: Move print_slabinfo_header to slab_common.c mm/sl[au]b: Move slabinfo processing to slab_common.c slub: remove one code path and reduce lock contention in __slab_free()
Diffstat (limited to 'mm')
-rw-r--r--mm/slab.c289
-rw-r--r--mm/slab.h53
-rw-r--r--mm/slab_common.c182
-rw-r--r--mm/slob.c46
-rw-r--r--mm/slub.c301
5 files changed, 387 insertions, 484 deletions
diff --git a/mm/slab.c b/mm/slab.c
index 33d3363658d..2c3a2e0394d 100644
--- a/mm/slab.c
+++ b/mm/slab.c
@@ -162,23 +162,6 @@
162 */ 162 */
163static bool pfmemalloc_active __read_mostly; 163static bool pfmemalloc_active __read_mostly;
164 164
165/* Legal flag mask for kmem_cache_create(). */
166#if DEBUG
167# define CREATE_MASK (SLAB_RED_ZONE | \
168 SLAB_POISON | SLAB_HWCACHE_ALIGN | \
169 SLAB_CACHE_DMA | \
170 SLAB_STORE_USER | \
171 SLAB_RECLAIM_ACCOUNT | SLAB_PANIC | \
172 SLAB_DESTROY_BY_RCU | SLAB_MEM_SPREAD | \
173 SLAB_DEBUG_OBJECTS | SLAB_NOLEAKTRACE | SLAB_NOTRACK)
174#else
175# define CREATE_MASK (SLAB_HWCACHE_ALIGN | \
176 SLAB_CACHE_DMA | \
177 SLAB_RECLAIM_ACCOUNT | SLAB_PANIC | \
178 SLAB_DESTROY_BY_RCU | SLAB_MEM_SPREAD | \
179 SLAB_DEBUG_OBJECTS | SLAB_NOLEAKTRACE | SLAB_NOTRACK)
180#endif
181
182/* 165/*
183 * kmem_bufctl_t: 166 * kmem_bufctl_t:
184 * 167 *
@@ -564,15 +547,11 @@ static struct cache_names __initdata cache_names[] = {
564#undef CACHE 547#undef CACHE
565}; 548};
566 549
567static struct arraycache_init initarray_cache __initdata =
568 { {0, BOOT_CPUCACHE_ENTRIES, 1, 0} };
569static struct arraycache_init initarray_generic = 550static struct arraycache_init initarray_generic =
570 { {0, BOOT_CPUCACHE_ENTRIES, 1, 0} }; 551 { {0, BOOT_CPUCACHE_ENTRIES, 1, 0} };
571 552
572/* internal cache of cache description objs */ 553/* internal cache of cache description objs */
573static struct kmem_list3 *kmem_cache_nodelists[MAX_NUMNODES];
574static struct kmem_cache kmem_cache_boot = { 554static struct kmem_cache kmem_cache_boot = {
575 .nodelists = kmem_cache_nodelists,
576 .batchcount = 1, 555 .batchcount = 1,
577 .limit = BOOT_CPUCACHE_ENTRIES, 556 .limit = BOOT_CPUCACHE_ENTRIES,
578 .shared = 1, 557 .shared = 1,
@@ -1577,28 +1556,33 @@ static void __init set_up_list3s(struct kmem_cache *cachep, int index)
1577} 1556}
1578 1557
1579/* 1558/*
1559 * The memory after the last cpu cache pointer is used for the
1560 * the nodelists pointer.
1561 */
1562static void setup_nodelists_pointer(struct kmem_cache *cachep)
1563{
1564 cachep->nodelists = (struct kmem_list3 **)&cachep->array[nr_cpu_ids];
1565}
1566
1567/*
1580 * Initialisation. Called after the page allocator have been initialised and 1568 * Initialisation. Called after the page allocator have been initialised and
1581 * before smp_init(). 1569 * before smp_init().
1582 */ 1570 */
1583void __init kmem_cache_init(void) 1571void __init kmem_cache_init(void)
1584{ 1572{
1585 size_t left_over;
1586 struct cache_sizes *sizes; 1573 struct cache_sizes *sizes;
1587 struct cache_names *names; 1574 struct cache_names *names;
1588 int i; 1575 int i;
1589 int order;
1590 int node;
1591 1576
1592 kmem_cache = &kmem_cache_boot; 1577 kmem_cache = &kmem_cache_boot;
1578 setup_nodelists_pointer(kmem_cache);
1593 1579
1594 if (num_possible_nodes() == 1) 1580 if (num_possible_nodes() == 1)
1595 use_alien_caches = 0; 1581 use_alien_caches = 0;
1596 1582
1597 for (i = 0; i < NUM_INIT_LISTS; i++) { 1583 for (i = 0; i < NUM_INIT_LISTS; i++)
1598 kmem_list3_init(&initkmem_list3[i]); 1584 kmem_list3_init(&initkmem_list3[i]);
1599 if (i < MAX_NUMNODES) 1585
1600 kmem_cache->nodelists[i] = NULL;
1601 }
1602 set_up_list3s(kmem_cache, CACHE_CACHE); 1586 set_up_list3s(kmem_cache, CACHE_CACHE);
1603 1587
1604 /* 1588 /*
@@ -1629,37 +1613,16 @@ void __init kmem_cache_init(void)
1629 * 6) Resize the head arrays of the kmalloc caches to their final sizes. 1613 * 6) Resize the head arrays of the kmalloc caches to their final sizes.
1630 */ 1614 */
1631 1615
1632 node = numa_mem_id();
1633
1634 /* 1) create the kmem_cache */ 1616 /* 1) create the kmem_cache */
1635 INIT_LIST_HEAD(&slab_caches);
1636 list_add(&kmem_cache->list, &slab_caches);
1637 kmem_cache->colour_off = cache_line_size();
1638 kmem_cache->array[smp_processor_id()] = &initarray_cache.cache;
1639 kmem_cache->nodelists[node] = &initkmem_list3[CACHE_CACHE + node];
1640 1617
1641 /* 1618 /*
1642 * struct kmem_cache size depends on nr_node_ids & nr_cpu_ids 1619 * struct kmem_cache size depends on nr_node_ids & nr_cpu_ids
1643 */ 1620 */
1644 kmem_cache->size = offsetof(struct kmem_cache, array[nr_cpu_ids]) + 1621 create_boot_cache(kmem_cache, "kmem_cache",
1645 nr_node_ids * sizeof(struct kmem_list3 *); 1622 offsetof(struct kmem_cache, array[nr_cpu_ids]) +
1646 kmem_cache->object_size = kmem_cache->size; 1623 nr_node_ids * sizeof(struct kmem_list3 *),
1647 kmem_cache->size = ALIGN(kmem_cache->object_size, 1624 SLAB_HWCACHE_ALIGN);
1648 cache_line_size()); 1625 list_add(&kmem_cache->list, &slab_caches);
1649 kmem_cache->reciprocal_buffer_size =
1650 reciprocal_value(kmem_cache->size);
1651
1652 for (order = 0; order < MAX_ORDER; order++) {
1653 cache_estimate(order, kmem_cache->size,
1654 cache_line_size(), 0, &left_over, &kmem_cache->num);
1655 if (kmem_cache->num)
1656 break;
1657 }
1658 BUG_ON(!kmem_cache->num);
1659 kmem_cache->gfporder = order;
1660 kmem_cache->colour = left_over / kmem_cache->colour_off;
1661 kmem_cache->slab_size = ALIGN(kmem_cache->num * sizeof(kmem_bufctl_t) +
1662 sizeof(struct slab), cache_line_size());
1663 1626
1664 /* 2+3) create the kmalloc caches */ 1627 /* 2+3) create the kmalloc caches */
1665 sizes = malloc_sizes; 1628 sizes = malloc_sizes;
@@ -1671,23 +1634,13 @@ void __init kmem_cache_init(void)
1671 * bug. 1634 * bug.
1672 */ 1635 */
1673 1636
1674 sizes[INDEX_AC].cs_cachep = kmem_cache_zalloc(kmem_cache, GFP_NOWAIT); 1637 sizes[INDEX_AC].cs_cachep = create_kmalloc_cache(names[INDEX_AC].name,
1675 sizes[INDEX_AC].cs_cachep->name = names[INDEX_AC].name; 1638 sizes[INDEX_AC].cs_size, ARCH_KMALLOC_FLAGS);
1676 sizes[INDEX_AC].cs_cachep->size = sizes[INDEX_AC].cs_size; 1639
1677 sizes[INDEX_AC].cs_cachep->object_size = sizes[INDEX_AC].cs_size; 1640 if (INDEX_AC != INDEX_L3)
1678 sizes[INDEX_AC].cs_cachep->align = ARCH_KMALLOC_MINALIGN; 1641 sizes[INDEX_L3].cs_cachep =
1679 __kmem_cache_create(sizes[INDEX_AC].cs_cachep, ARCH_KMALLOC_FLAGS|SLAB_PANIC); 1642 create_kmalloc_cache(names[INDEX_L3].name,
1680 list_add(&sizes[INDEX_AC].cs_cachep->list, &slab_caches); 1643 sizes[INDEX_L3].cs_size, ARCH_KMALLOC_FLAGS);
1681
1682 if (INDEX_AC != INDEX_L3) {
1683 sizes[INDEX_L3].cs_cachep = kmem_cache_zalloc(kmem_cache, GFP_NOWAIT);
1684 sizes[INDEX_L3].cs_cachep->name = names[INDEX_L3].name;
1685 sizes[INDEX_L3].cs_cachep->size = sizes[INDEX_L3].cs_size;
1686 sizes[INDEX_L3].cs_cachep->object_size = sizes[INDEX_L3].cs_size;
1687 sizes[INDEX_L3].cs_cachep->align = ARCH_KMALLOC_MINALIGN;
1688 __kmem_cache_create(sizes[INDEX_L3].cs_cachep, ARCH_KMALLOC_FLAGS|SLAB_PANIC);
1689 list_add(&sizes[INDEX_L3].cs_cachep->list, &slab_caches);
1690 }
1691 1644
1692 slab_early_init = 0; 1645 slab_early_init = 0;
1693 1646
@@ -1699,24 +1652,14 @@ void __init kmem_cache_init(void)
1699 * Note for systems short on memory removing the alignment will 1652 * Note for systems short on memory removing the alignment will
1700 * allow tighter packing of the smaller caches. 1653 * allow tighter packing of the smaller caches.
1701 */ 1654 */
1702 if (!sizes->cs_cachep) { 1655 if (!sizes->cs_cachep)
1703 sizes->cs_cachep = kmem_cache_zalloc(kmem_cache, GFP_NOWAIT); 1656 sizes->cs_cachep = create_kmalloc_cache(names->name,
1704 sizes->cs_cachep->name = names->name; 1657 sizes->cs_size, ARCH_KMALLOC_FLAGS);
1705 sizes->cs_cachep->size = sizes->cs_size; 1658
1706 sizes->cs_cachep->object_size = sizes->cs_size;
1707 sizes->cs_cachep->align = ARCH_KMALLOC_MINALIGN;
1708 __kmem_cache_create(sizes->cs_cachep, ARCH_KMALLOC_FLAGS|SLAB_PANIC);
1709 list_add(&sizes->cs_cachep->list, &slab_caches);
1710 }
1711#ifdef CONFIG_ZONE_DMA 1659#ifdef CONFIG_ZONE_DMA
1712 sizes->cs_dmacachep = kmem_cache_zalloc(kmem_cache, GFP_NOWAIT); 1660 sizes->cs_dmacachep = create_kmalloc_cache(
1713 sizes->cs_dmacachep->name = names->name_dma; 1661 names->name_dma, sizes->cs_size,
1714 sizes->cs_dmacachep->size = sizes->cs_size; 1662 SLAB_CACHE_DMA|ARCH_KMALLOC_FLAGS);
1715 sizes->cs_dmacachep->object_size = sizes->cs_size;
1716 sizes->cs_dmacachep->align = ARCH_KMALLOC_MINALIGN;
1717 __kmem_cache_create(sizes->cs_dmacachep,
1718 ARCH_KMALLOC_FLAGS|SLAB_CACHE_DMA| SLAB_PANIC);
1719 list_add(&sizes->cs_dmacachep->list, &slab_caches);
1720#endif 1663#endif
1721 sizes++; 1664 sizes++;
1722 names++; 1665 names++;
@@ -1727,7 +1670,6 @@ void __init kmem_cache_init(void)
1727 1670
1728 ptr = kmalloc(sizeof(struct arraycache_init), GFP_NOWAIT); 1671 ptr = kmalloc(sizeof(struct arraycache_init), GFP_NOWAIT);
1729 1672
1730 BUG_ON(cpu_cache_get(kmem_cache) != &initarray_cache.cache);
1731 memcpy(ptr, cpu_cache_get(kmem_cache), 1673 memcpy(ptr, cpu_cache_get(kmem_cache),
1732 sizeof(struct arraycache_init)); 1674 sizeof(struct arraycache_init));
1733 /* 1675 /*
@@ -2282,7 +2224,15 @@ static int __init_refok setup_cpu_cache(struct kmem_cache *cachep, gfp_t gfp)
2282 2224
2283 if (slab_state == DOWN) { 2225 if (slab_state == DOWN) {
2284 /* 2226 /*
2285 * Note: the first kmem_cache_create must create the cache 2227 * Note: Creation of first cache (kmem_cache).
2228 * The setup_list3s is taken care
2229 * of by the caller of __kmem_cache_create
2230 */
2231 cachep->array[smp_processor_id()] = &initarray_generic.cache;
2232 slab_state = PARTIAL;
2233 } else if (slab_state == PARTIAL) {
2234 /*
2235 * Note: the second kmem_cache_create must create the cache
2286 * that's used by kmalloc(24), otherwise the creation of 2236 * that's used by kmalloc(24), otherwise the creation of
2287 * further caches will BUG(). 2237 * further caches will BUG().
2288 */ 2238 */
@@ -2290,7 +2240,7 @@ static int __init_refok setup_cpu_cache(struct kmem_cache *cachep, gfp_t gfp)
2290 2240
2291 /* 2241 /*
2292 * If the cache that's used by kmalloc(sizeof(kmem_list3)) is 2242 * If the cache that's used by kmalloc(sizeof(kmem_list3)) is
2293 * the first cache, then we need to set up all its list3s, 2243 * the second cache, then we need to set up all its list3s,
2294 * otherwise the creation of further caches will BUG(). 2244 * otherwise the creation of further caches will BUG().
2295 */ 2245 */
2296 set_up_list3s(cachep, SIZE_AC); 2246 set_up_list3s(cachep, SIZE_AC);
@@ -2299,6 +2249,7 @@ static int __init_refok setup_cpu_cache(struct kmem_cache *cachep, gfp_t gfp)
2299 else 2249 else
2300 slab_state = PARTIAL_ARRAYCACHE; 2250 slab_state = PARTIAL_ARRAYCACHE;
2301 } else { 2251 } else {
2252 /* Remaining boot caches */
2302 cachep->array[smp_processor_id()] = 2253 cachep->array[smp_processor_id()] =
2303 kmalloc(sizeof(struct arraycache_init), gfp); 2254 kmalloc(sizeof(struct arraycache_init), gfp);
2304 2255
@@ -2331,11 +2282,8 @@ static int __init_refok setup_cpu_cache(struct kmem_cache *cachep, gfp_t gfp)
2331 2282
2332/** 2283/**
2333 * __kmem_cache_create - Create a cache. 2284 * __kmem_cache_create - Create a cache.
2334 * @name: A string which is used in /proc/slabinfo to identify this cache. 2285 * @cachep: cache management descriptor
2335 * @size: The size of objects to be created in this cache.
2336 * @align: The required alignment for the objects.
2337 * @flags: SLAB flags 2286 * @flags: SLAB flags
2338 * @ctor: A constructor for the objects.
2339 * 2287 *
2340 * Returns a ptr to the cache on success, NULL on failure. 2288 * Returns a ptr to the cache on success, NULL on failure.
2341 * Cannot be called within a int, but can be interrupted. 2289 * Cannot be called within a int, but can be interrupted.
@@ -2378,11 +2326,6 @@ __kmem_cache_create (struct kmem_cache *cachep, unsigned long flags)
2378 if (flags & SLAB_DESTROY_BY_RCU) 2326 if (flags & SLAB_DESTROY_BY_RCU)
2379 BUG_ON(flags & SLAB_POISON); 2327 BUG_ON(flags & SLAB_POISON);
2380#endif 2328#endif
2381 /*
2382 * Always checks flags, a caller might be expecting debug support which
2383 * isn't available.
2384 */
2385 BUG_ON(flags & ~CREATE_MASK);
2386 2329
2387 /* 2330 /*
2388 * Check that size is in terms of words. This is needed to avoid 2331 * Check that size is in terms of words. This is needed to avoid
@@ -2394,22 +2337,6 @@ __kmem_cache_create (struct kmem_cache *cachep, unsigned long flags)
2394 size &= ~(BYTES_PER_WORD - 1); 2337 size &= ~(BYTES_PER_WORD - 1);
2395 } 2338 }
2396 2339
2397 /* calculate the final buffer alignment: */
2398
2399 /* 1) arch recommendation: can be overridden for debug */
2400 if (flags & SLAB_HWCACHE_ALIGN) {
2401 /*
2402 * Default alignment: as specified by the arch code. Except if
2403 * an object is really small, then squeeze multiple objects into
2404 * one cacheline.
2405 */
2406 ralign = cache_line_size();
2407 while (size <= ralign / 2)
2408 ralign /= 2;
2409 } else {
2410 ralign = BYTES_PER_WORD;
2411 }
2412
2413 /* 2340 /*
2414 * Redzoning and user store require word alignment or possibly larger. 2341 * Redzoning and user store require word alignment or possibly larger.
2415 * Note this will be overridden by architecture or caller mandated 2342 * Note this will be overridden by architecture or caller mandated
@@ -2426,10 +2353,6 @@ __kmem_cache_create (struct kmem_cache *cachep, unsigned long flags)
2426 size &= ~(REDZONE_ALIGN - 1); 2353 size &= ~(REDZONE_ALIGN - 1);
2427 } 2354 }
2428 2355
2429 /* 2) arch mandated alignment */
2430 if (ralign < ARCH_SLAB_MINALIGN) {
2431 ralign = ARCH_SLAB_MINALIGN;
2432 }
2433 /* 3) caller mandated alignment */ 2356 /* 3) caller mandated alignment */
2434 if (ralign < cachep->align) { 2357 if (ralign < cachep->align) {
2435 ralign = cachep->align; 2358 ralign = cachep->align;
@@ -2447,7 +2370,7 @@ __kmem_cache_create (struct kmem_cache *cachep, unsigned long flags)
2447 else 2370 else
2448 gfp = GFP_NOWAIT; 2371 gfp = GFP_NOWAIT;
2449 2372
2450 cachep->nodelists = (struct kmem_list3 **)&cachep->array[nr_cpu_ids]; 2373 setup_nodelists_pointer(cachep);
2451#if DEBUG 2374#if DEBUG
2452 2375
2453 /* 2376 /*
@@ -3969,12 +3892,6 @@ void kfree(const void *objp)
3969} 3892}
3970EXPORT_SYMBOL(kfree); 3893EXPORT_SYMBOL(kfree);
3971 3894
3972unsigned int kmem_cache_size(struct kmem_cache *cachep)
3973{
3974 return cachep->object_size;
3975}
3976EXPORT_SYMBOL(kmem_cache_size);
3977
3978/* 3895/*
3979 * This initializes kmem_list3 or resizes various caches for all nodes. 3896 * This initializes kmem_list3 or resizes various caches for all nodes.
3980 */ 3897 */
@@ -4276,54 +4193,8 @@ out:
4276} 4193}
4277 4194
4278#ifdef CONFIG_SLABINFO 4195#ifdef CONFIG_SLABINFO
4279 4196void get_slabinfo(struct kmem_cache *cachep, struct slabinfo *sinfo)
4280static void print_slabinfo_header(struct seq_file *m)
4281{ 4197{
4282 /*
4283 * Output format version, so at least we can change it
4284 * without _too_ many complaints.
4285 */
4286#if STATS
4287 seq_puts(m, "slabinfo - version: 2.1 (statistics)\n");
4288#else
4289 seq_puts(m, "slabinfo - version: 2.1\n");
4290#endif
4291 seq_puts(m, "# name <active_objs> <num_objs> <objsize> "
4292 "<objperslab> <pagesperslab>");
4293 seq_puts(m, " : tunables <limit> <batchcount> <sharedfactor>");
4294 seq_puts(m, " : slabdata <active_slabs> <num_slabs> <sharedavail>");
4295#if STATS
4296 seq_puts(m, " : globalstat <listallocs> <maxobjs> <grown> <reaped> "
4297 "<error> <maxfreeable> <nodeallocs> <remotefrees> <alienoverflow>");
4298 seq_puts(m, " : cpustat <allochit> <allocmiss> <freehit> <freemiss>");
4299#endif
4300 seq_putc(m, '\n');
4301}
4302
4303static void *s_start(struct seq_file *m, loff_t *pos)
4304{
4305 loff_t n = *pos;
4306
4307 mutex_lock(&slab_mutex);
4308 if (!n)
4309 print_slabinfo_header(m);
4310
4311 return seq_list_start(&slab_caches, *pos);
4312}
4313
4314static void *s_next(struct seq_file *m, void *p, loff_t *pos)
4315{
4316 return seq_list_next(p, &slab_caches, pos);
4317}
4318
4319static void s_stop(struct seq_file *m, void *p)
4320{
4321 mutex_unlock(&slab_mutex);
4322}
4323
4324static int s_show(struct seq_file *m, void *p)
4325{
4326 struct kmem_cache *cachep = list_entry(p, struct kmem_cache, list);
4327 struct slab *slabp; 4198 struct slab *slabp;
4328 unsigned long active_objs; 4199 unsigned long active_objs;
4329 unsigned long num_objs; 4200 unsigned long num_objs;
@@ -4378,13 +4249,20 @@ static int s_show(struct seq_file *m, void *p)
4378 if (error) 4249 if (error)
4379 printk(KERN_ERR "slab: cache %s error: %s\n", name, error); 4250 printk(KERN_ERR "slab: cache %s error: %s\n", name, error);
4380 4251
4381 seq_printf(m, "%-17s %6lu %6lu %6u %4u %4d", 4252 sinfo->active_objs = active_objs;
4382 name, active_objs, num_objs, cachep->size, 4253 sinfo->num_objs = num_objs;
4383 cachep->num, (1 << cachep->gfporder)); 4254 sinfo->active_slabs = active_slabs;
4384 seq_printf(m, " : tunables %4u %4u %4u", 4255 sinfo->num_slabs = num_slabs;
4385 cachep->limit, cachep->batchcount, cachep->shared); 4256 sinfo->shared_avail = shared_avail;
4386 seq_printf(m, " : slabdata %6lu %6lu %6lu", 4257 sinfo->limit = cachep->limit;
4387 active_slabs, num_slabs, shared_avail); 4258 sinfo->batchcount = cachep->batchcount;
4259 sinfo->shared = cachep->shared;
4260 sinfo->objects_per_slab = cachep->num;
4261 sinfo->cache_order = cachep->gfporder;
4262}
4263
4264void slabinfo_show_stats(struct seq_file *m, struct kmem_cache *cachep)
4265{
4388#if STATS 4266#if STATS
4389 { /* list3 stats */ 4267 { /* list3 stats */
4390 unsigned long high = cachep->high_mark; 4268 unsigned long high = cachep->high_mark;
@@ -4414,31 +4292,8 @@ static int s_show(struct seq_file *m, void *p)
4414 allochit, allocmiss, freehit, freemiss); 4292 allochit, allocmiss, freehit, freemiss);
4415 } 4293 }
4416#endif 4294#endif
4417 seq_putc(m, '\n');
4418 return 0;
4419} 4295}
4420 4296
4421/*
4422 * slabinfo_op - iterator that generates /proc/slabinfo
4423 *
4424 * Output layout:
4425 * cache-name
4426 * num-active-objs
4427 * total-objs
4428 * object size
4429 * num-active-slabs
4430 * total-slabs
4431 * num-pages-per-slab
4432 * + further values on SMP and with statistics enabled
4433 */
4434
4435static const struct seq_operations slabinfo_op = {
4436 .start = s_start,
4437 .next = s_next,
4438 .stop = s_stop,
4439 .show = s_show,
4440};
4441
4442#define MAX_SLABINFO_WRITE 128 4297#define MAX_SLABINFO_WRITE 128
4443/** 4298/**
4444 * slabinfo_write - Tuning for the slab allocator 4299 * slabinfo_write - Tuning for the slab allocator
@@ -4447,7 +4302,7 @@ static const struct seq_operations slabinfo_op = {
4447 * @count: data length 4302 * @count: data length
4448 * @ppos: unused 4303 * @ppos: unused
4449 */ 4304 */
4450static ssize_t slabinfo_write(struct file *file, const char __user *buffer, 4305ssize_t slabinfo_write(struct file *file, const char __user *buffer,
4451 size_t count, loff_t *ppos) 4306 size_t count, loff_t *ppos)
4452{ 4307{
4453 char kbuf[MAX_SLABINFO_WRITE + 1], *tmp; 4308 char kbuf[MAX_SLABINFO_WRITE + 1], *tmp;
@@ -4490,19 +4345,6 @@ static ssize_t slabinfo_write(struct file *file, const char __user *buffer,
4490 return res; 4345 return res;
4491} 4346}
4492 4347
4493static int slabinfo_open(struct inode *inode, struct file *file)
4494{
4495 return seq_open(file, &slabinfo_op);
4496}
4497
4498static const struct file_operations proc_slabinfo_operations = {
4499 .open = slabinfo_open,
4500 .read = seq_read,
4501 .write = slabinfo_write,
4502 .llseek = seq_lseek,
4503 .release = seq_release,
4504};
4505
4506#ifdef CONFIG_DEBUG_SLAB_LEAK 4348#ifdef CONFIG_DEBUG_SLAB_LEAK
4507 4349
4508static void *leaks_start(struct seq_file *m, loff_t *pos) 4350static void *leaks_start(struct seq_file *m, loff_t *pos)
@@ -4631,6 +4473,16 @@ static int leaks_show(struct seq_file *m, void *p)
4631 return 0; 4473 return 0;
4632} 4474}
4633 4475
4476static void *s_next(struct seq_file *m, void *p, loff_t *pos)
4477{
4478 return seq_list_next(p, &slab_caches, pos);
4479}
4480
4481static void s_stop(struct seq_file *m, void *p)
4482{
4483 mutex_unlock(&slab_mutex);
4484}
4485
4634static const struct seq_operations slabstats_op = { 4486static const struct seq_operations slabstats_op = {
4635 .start = leaks_start, 4487 .start = leaks_start,
4636 .next = s_next, 4488 .next = s_next,
@@ -4665,7 +4517,6 @@ static const struct file_operations proc_slabstats_operations = {
4665 4517
4666static int __init slab_proc_init(void) 4518static int __init slab_proc_init(void)
4667{ 4519{
4668 proc_create("slabinfo",S_IWUSR|S_IRUSR,NULL,&proc_slabinfo_operations);
4669#ifdef CONFIG_DEBUG_SLAB_LEAK 4520#ifdef CONFIG_DEBUG_SLAB_LEAK
4670 proc_create("slab_allocators", 0, NULL, &proc_slabstats_operations); 4521 proc_create("slab_allocators", 0, NULL, &proc_slabstats_operations);
4671#endif 4522#endif
diff --git a/mm/slab.h b/mm/slab.h
index 7deeb449a30..1cb9c9ee0e6 100644
--- a/mm/slab.h
+++ b/mm/slab.h
@@ -32,9 +32,17 @@ extern struct list_head slab_caches;
32/* The slab cache that manages slab cache information */ 32/* The slab cache that manages slab cache information */
33extern struct kmem_cache *kmem_cache; 33extern struct kmem_cache *kmem_cache;
34 34
35unsigned long calculate_alignment(unsigned long flags,
36 unsigned long align, unsigned long size);
37
35/* Functions provided by the slab allocators */ 38/* Functions provided by the slab allocators */
36extern int __kmem_cache_create(struct kmem_cache *, unsigned long flags); 39extern int __kmem_cache_create(struct kmem_cache *, unsigned long flags);
37 40
41extern struct kmem_cache *create_kmalloc_cache(const char *name, size_t size,
42 unsigned long flags);
43extern void create_boot_cache(struct kmem_cache *, const char *name,
44 size_t size, unsigned long flags);
45
38#ifdef CONFIG_SLUB 46#ifdef CONFIG_SLUB
39struct kmem_cache *__kmem_cache_alias(const char *name, size_t size, 47struct kmem_cache *__kmem_cache_alias(const char *name, size_t size,
40 size_t align, unsigned long flags, void (*ctor)(void *)); 48 size_t align, unsigned long flags, void (*ctor)(void *));
@@ -45,6 +53,51 @@ static inline struct kmem_cache *__kmem_cache_alias(const char *name, size_t siz
45#endif 53#endif
46 54
47 55
56/* Legal flag mask for kmem_cache_create(), for various configurations */
57#define SLAB_CORE_FLAGS (SLAB_HWCACHE_ALIGN | SLAB_CACHE_DMA | SLAB_PANIC | \
58 SLAB_DESTROY_BY_RCU | SLAB_DEBUG_OBJECTS )
59
60#if defined(CONFIG_DEBUG_SLAB)
61#define SLAB_DEBUG_FLAGS (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER)
62#elif defined(CONFIG_SLUB_DEBUG)
63#define SLAB_DEBUG_FLAGS (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER | \
64 SLAB_TRACE | SLAB_DEBUG_FREE)
65#else
66#define SLAB_DEBUG_FLAGS (0)
67#endif
68
69#if defined(CONFIG_SLAB)
70#define SLAB_CACHE_FLAGS (SLAB_MEM_SPREAD | SLAB_NOLEAKTRACE | \
71 SLAB_RECLAIM_ACCOUNT | SLAB_TEMPORARY | SLAB_NOTRACK)
72#elif defined(CONFIG_SLUB)
73#define SLAB_CACHE_FLAGS (SLAB_NOLEAKTRACE | SLAB_RECLAIM_ACCOUNT | \
74 SLAB_TEMPORARY | SLAB_NOTRACK)
75#else
76#define SLAB_CACHE_FLAGS (0)
77#endif
78
79#define CACHE_CREATE_MASK (SLAB_CORE_FLAGS | SLAB_DEBUG_FLAGS | SLAB_CACHE_FLAGS)
80
48int __kmem_cache_shutdown(struct kmem_cache *); 81int __kmem_cache_shutdown(struct kmem_cache *);
49 82
83struct seq_file;
84struct file;
85
86struct slabinfo {
87 unsigned long active_objs;
88 unsigned long num_objs;
89 unsigned long active_slabs;
90 unsigned long num_slabs;
91 unsigned long shared_avail;
92 unsigned int limit;
93 unsigned int batchcount;
94 unsigned int shared;
95 unsigned int objects_per_slab;
96 unsigned int cache_order;
97};
98
99void get_slabinfo(struct kmem_cache *s, struct slabinfo *sinfo);
100void slabinfo_show_stats(struct seq_file *m, struct kmem_cache *s);
101ssize_t slabinfo_write(struct file *file, const char __user *buffer,
102 size_t count, loff_t *ppos);
50#endif 103#endif
diff --git a/mm/slab_common.c b/mm/slab_common.c
index 069a24e6440..a8e76d79ee6 100644
--- a/mm/slab_common.c
+++ b/mm/slab_common.c
@@ -13,6 +13,8 @@
13#include <linux/module.h> 13#include <linux/module.h>
14#include <linux/cpu.h> 14#include <linux/cpu.h>
15#include <linux/uaccess.h> 15#include <linux/uaccess.h>
16#include <linux/seq_file.h>
17#include <linux/proc_fs.h>
16#include <asm/cacheflush.h> 18#include <asm/cacheflush.h>
17#include <asm/tlbflush.h> 19#include <asm/tlbflush.h>
18#include <asm/page.h> 20#include <asm/page.h>
@@ -71,6 +73,34 @@ static inline int kmem_cache_sanity_check(const char *name, size_t size)
71#endif 73#endif
72 74
73/* 75/*
76 * Figure out what the alignment of the objects will be given a set of
77 * flags, a user specified alignment and the size of the objects.
78 */
79unsigned long calculate_alignment(unsigned long flags,
80 unsigned long align, unsigned long size)
81{
82 /*
83 * If the user wants hardware cache aligned objects then follow that
84 * suggestion if the object is sufficiently large.
85 *
86 * The hardware cache alignment cannot override the specified
87 * alignment though. If that is greater then use it.
88 */
89 if (flags & SLAB_HWCACHE_ALIGN) {
90 unsigned long ralign = cache_line_size();
91 while (size <= ralign / 2)
92 ralign /= 2;
93 align = max(align, ralign);
94 }
95
96 if (align < ARCH_SLAB_MINALIGN)
97 align = ARCH_SLAB_MINALIGN;
98
99 return ALIGN(align, sizeof(void *));
100}
101
102
103/*
74 * kmem_cache_create - Create a cache. 104 * kmem_cache_create - Create a cache.
75 * @name: A string which is used in /proc/slabinfo to identify this cache. 105 * @name: A string which is used in /proc/slabinfo to identify this cache.
76 * @size: The size of objects to be created in this cache. 106 * @size: The size of objects to be created in this cache.
@@ -107,6 +137,13 @@ struct kmem_cache *kmem_cache_create(const char *name, size_t size, size_t align
107 if (!kmem_cache_sanity_check(name, size) == 0) 137 if (!kmem_cache_sanity_check(name, size) == 0)
108 goto out_locked; 138 goto out_locked;
109 139
140 /*
141 * Some allocators will constraint the set of valid flags to a subset
142 * of all flags. We expect them to define CACHE_CREATE_MASK in this
143 * case, and we'll just provide them with a sanitized version of the
144 * passed flags.
145 */
146 flags &= CACHE_CREATE_MASK;
110 147
111 s = __kmem_cache_alias(name, size, align, flags, ctor); 148 s = __kmem_cache_alias(name, size, align, flags, ctor);
112 if (s) 149 if (s)
@@ -115,7 +152,7 @@ struct kmem_cache *kmem_cache_create(const char *name, size_t size, size_t align
115 s = kmem_cache_zalloc(kmem_cache, GFP_KERNEL); 152 s = kmem_cache_zalloc(kmem_cache, GFP_KERNEL);
116 if (s) { 153 if (s) {
117 s->object_size = s->size = size; 154 s->object_size = s->size = size;
118 s->align = align; 155 s->align = calculate_alignment(flags, align, size);
119 s->ctor = ctor; 156 s->ctor = ctor;
120 s->name = kstrdup(name, GFP_KERNEL); 157 s->name = kstrdup(name, GFP_KERNEL);
121 if (!s->name) { 158 if (!s->name) {
@@ -192,3 +229,146 @@ int slab_is_available(void)
192{ 229{
193 return slab_state >= UP; 230 return slab_state >= UP;
194} 231}
232
233#ifndef CONFIG_SLOB
234/* Create a cache during boot when no slab services are available yet */
235void __init create_boot_cache(struct kmem_cache *s, const char *name, size_t size,
236 unsigned long flags)
237{
238 int err;
239
240 s->name = name;
241 s->size = s->object_size = size;
242 s->align = calculate_alignment(flags, ARCH_KMALLOC_MINALIGN, size);
243 err = __kmem_cache_create(s, flags);
244
245 if (err)
246 panic("Creation of kmalloc slab %s size=%zd failed. Reason %d\n",
247 name, size, err);
248
249 s->refcount = -1; /* Exempt from merging for now */
250}
251
252struct kmem_cache *__init create_kmalloc_cache(const char *name, size_t size,
253 unsigned long flags)
254{
255 struct kmem_cache *s = kmem_cache_zalloc(kmem_cache, GFP_NOWAIT);
256
257 if (!s)
258 panic("Out of memory when creating slab %s\n", name);
259
260 create_boot_cache(s, name, size, flags);
261 list_add(&s->list, &slab_caches);
262 s->refcount = 1;
263 return s;
264}
265
266#endif /* !CONFIG_SLOB */
267
268
269#ifdef CONFIG_SLABINFO
270static void print_slabinfo_header(struct seq_file *m)
271{
272 /*
273 * Output format version, so at least we can change it
274 * without _too_ many complaints.
275 */
276#ifdef CONFIG_DEBUG_SLAB
277 seq_puts(m, "slabinfo - version: 2.1 (statistics)\n");
278#else
279 seq_puts(m, "slabinfo - version: 2.1\n");
280#endif
281 seq_puts(m, "# name <active_objs> <num_objs> <objsize> "
282 "<objperslab> <pagesperslab>");
283 seq_puts(m, " : tunables <limit> <batchcount> <sharedfactor>");
284 seq_puts(m, " : slabdata <active_slabs> <num_slabs> <sharedavail>");
285#ifdef CONFIG_DEBUG_SLAB
286 seq_puts(m, " : globalstat <listallocs> <maxobjs> <grown> <reaped> "
287 "<error> <maxfreeable> <nodeallocs> <remotefrees> <alienoverflow>");
288 seq_puts(m, " : cpustat <allochit> <allocmiss> <freehit> <freemiss>");
289#endif
290 seq_putc(m, '\n');
291}
292
293static void *s_start(struct seq_file *m, loff_t *pos)
294{
295 loff_t n = *pos;
296
297 mutex_lock(&slab_mutex);
298 if (!n)
299 print_slabinfo_header(m);
300
301 return seq_list_start(&slab_caches, *pos);
302}
303
304static void *s_next(struct seq_file *m, void *p, loff_t *pos)
305{
306 return seq_list_next(p, &slab_caches, pos);
307}
308
309static void s_stop(struct seq_file *m, void *p)
310{
311 mutex_unlock(&slab_mutex);
312}
313
314static int s_show(struct seq_file *m, void *p)
315{
316 struct kmem_cache *s = list_entry(p, struct kmem_cache, list);
317 struct slabinfo sinfo;
318
319 memset(&sinfo, 0, sizeof(sinfo));
320 get_slabinfo(s, &sinfo);
321
322 seq_printf(m, "%-17s %6lu %6lu %6u %4u %4d",
323 s->name, sinfo.active_objs, sinfo.num_objs, s->size,
324 sinfo.objects_per_slab, (1 << sinfo.cache_order));
325
326 seq_printf(m, " : tunables %4u %4u %4u",
327 sinfo.limit, sinfo.batchcount, sinfo.shared);
328 seq_printf(m, " : slabdata %6lu %6lu %6lu",
329 sinfo.active_slabs, sinfo.num_slabs, sinfo.shared_avail);
330 slabinfo_show_stats(m, s);
331 seq_putc(m, '\n');
332 return 0;
333}
334
335/*
336 * slabinfo_op - iterator that generates /proc/slabinfo
337 *
338 * Output layout:
339 * cache-name
340 * num-active-objs
341 * total-objs
342 * object size
343 * num-active-slabs
344 * total-slabs
345 * num-pages-per-slab
346 * + further values on SMP and with statistics enabled
347 */
348static const struct seq_operations slabinfo_op = {
349 .start = s_start,
350 .next = s_next,
351 .stop = s_stop,
352 .show = s_show,
353};
354
355static int slabinfo_open(struct inode *inode, struct file *file)
356{
357 return seq_open(file, &slabinfo_op);
358}
359
360static const struct file_operations proc_slabinfo_operations = {
361 .open = slabinfo_open,
362 .read = seq_read,
363 .write = slabinfo_write,
364 .llseek = seq_lseek,
365 .release = seq_release,
366};
367
368static int __init slab_proc_init(void)
369{
370 proc_create("slabinfo", S_IRUSR, NULL, &proc_slabinfo_operations);
371 return 0;
372}
373module_init(slab_proc_init);
374#endif /* CONFIG_SLABINFO */
diff --git a/mm/slob.c b/mm/slob.c
index 1e921c5e957..795bab7d391 100644
--- a/mm/slob.c
+++ b/mm/slob.c
@@ -28,9 +28,8 @@
28 * from kmalloc are prepended with a 4-byte header with the kmalloc size. 28 * from kmalloc are prepended with a 4-byte header with the kmalloc size.
29 * If kmalloc is asked for objects of PAGE_SIZE or larger, it calls 29 * If kmalloc is asked for objects of PAGE_SIZE or larger, it calls
30 * alloc_pages() directly, allocating compound pages so the page order 30 * alloc_pages() directly, allocating compound pages so the page order
31 * does not have to be separately tracked, and also stores the exact 31 * does not have to be separately tracked.
32 * allocation size in page->private so that it can be used to accurately 32 * These objects are detected in kfree() because PageSlab()
33 * provide ksize(). These objects are detected in kfree() because slob_page()
34 * is false for them. 33 * is false for them.
35 * 34 *
36 * SLAB is emulated on top of SLOB by simply calling constructors and 35 * SLAB is emulated on top of SLOB by simply calling constructors and
@@ -124,7 +123,6 @@ static inline void clear_slob_page_free(struct page *sp)
124 123
125#define SLOB_UNIT sizeof(slob_t) 124#define SLOB_UNIT sizeof(slob_t)
126#define SLOB_UNITS(size) (((size) + SLOB_UNIT - 1)/SLOB_UNIT) 125#define SLOB_UNITS(size) (((size) + SLOB_UNIT - 1)/SLOB_UNIT)
127#define SLOB_ALIGN L1_CACHE_BYTES
128 126
129/* 127/*
130 * struct slob_rcu is inserted at the tail of allocated slob blocks, which 128 * struct slob_rcu is inserted at the tail of allocated slob blocks, which
@@ -455,11 +453,6 @@ __do_kmalloc_node(size_t size, gfp_t gfp, int node, unsigned long caller)
455 if (likely(order)) 453 if (likely(order))
456 gfp |= __GFP_COMP; 454 gfp |= __GFP_COMP;
457 ret = slob_new_pages(gfp, order, node); 455 ret = slob_new_pages(gfp, order, node);
458 if (ret) {
459 struct page *page;
460 page = virt_to_page(ret);
461 page->private = size;
462 }
463 456
464 trace_kmalloc_node(caller, ret, 457 trace_kmalloc_node(caller, ret,
465 size, PAGE_SIZE << order, gfp, node); 458 size, PAGE_SIZE << order, gfp, node);
@@ -506,7 +499,7 @@ void kfree(const void *block)
506 unsigned int *m = (unsigned int *)(block - align); 499 unsigned int *m = (unsigned int *)(block - align);
507 slob_free(m, *m + align); 500 slob_free(m, *m + align);
508 } else 501 } else
509 put_page(sp); 502 __free_pages(sp, compound_order(sp));
510} 503}
511EXPORT_SYMBOL(kfree); 504EXPORT_SYMBOL(kfree);
512 505
@@ -514,37 +507,30 @@ EXPORT_SYMBOL(kfree);
514size_t ksize(const void *block) 507size_t ksize(const void *block)
515{ 508{
516 struct page *sp; 509 struct page *sp;
510 int align;
511 unsigned int *m;
517 512
518 BUG_ON(!block); 513 BUG_ON(!block);
519 if (unlikely(block == ZERO_SIZE_PTR)) 514 if (unlikely(block == ZERO_SIZE_PTR))
520 return 0; 515 return 0;
521 516
522 sp = virt_to_page(block); 517 sp = virt_to_page(block);
523 if (PageSlab(sp)) { 518 if (unlikely(!PageSlab(sp)))
524 int align = max_t(size_t, ARCH_KMALLOC_MINALIGN, ARCH_SLAB_MINALIGN); 519 return PAGE_SIZE << compound_order(sp);
525 unsigned int *m = (unsigned int *)(block - align); 520
526 return SLOB_UNITS(*m) * SLOB_UNIT; 521 align = max_t(size_t, ARCH_KMALLOC_MINALIGN, ARCH_SLAB_MINALIGN);
527 } else 522 m = (unsigned int *)(block - align);
528 return sp->private; 523 return SLOB_UNITS(*m) * SLOB_UNIT;
529} 524}
530EXPORT_SYMBOL(ksize); 525EXPORT_SYMBOL(ksize);
531 526
532int __kmem_cache_create(struct kmem_cache *c, unsigned long flags) 527int __kmem_cache_create(struct kmem_cache *c, unsigned long flags)
533{ 528{
534 size_t align = c->size;
535
536 if (flags & SLAB_DESTROY_BY_RCU) { 529 if (flags & SLAB_DESTROY_BY_RCU) {
537 /* leave room for rcu footer at the end of object */ 530 /* leave room for rcu footer at the end of object */
538 c->size += sizeof(struct slob_rcu); 531 c->size += sizeof(struct slob_rcu);
539 } 532 }
540 c->flags = flags; 533 c->flags = flags;
541 /* ignore alignment unless it's forced */
542 c->align = (flags & SLAB_HWCACHE_ALIGN) ? SLOB_ALIGN : 0;
543 if (c->align < ARCH_SLAB_MINALIGN)
544 c->align = ARCH_SLAB_MINALIGN;
545 if (c->align < align)
546 c->align = align;
547
548 return 0; 534 return 0;
549} 535}
550 536
@@ -558,12 +544,12 @@ void *kmem_cache_alloc_node(struct kmem_cache *c, gfp_t flags, int node)
558 544
559 if (c->size < PAGE_SIZE) { 545 if (c->size < PAGE_SIZE) {
560 b = slob_alloc(c->size, flags, c->align, node); 546 b = slob_alloc(c->size, flags, c->align, node);
561 trace_kmem_cache_alloc_node(_RET_IP_, b, c->size, 547 trace_kmem_cache_alloc_node(_RET_IP_, b, c->object_size,
562 SLOB_UNITS(c->size) * SLOB_UNIT, 548 SLOB_UNITS(c->size) * SLOB_UNIT,
563 flags, node); 549 flags, node);
564 } else { 550 } else {
565 b = slob_new_pages(flags, get_order(c->size), node); 551 b = slob_new_pages(flags, get_order(c->size), node);
566 trace_kmem_cache_alloc_node(_RET_IP_, b, c->size, 552 trace_kmem_cache_alloc_node(_RET_IP_, b, c->object_size,
567 PAGE_SIZE << get_order(c->size), 553 PAGE_SIZE << get_order(c->size),
568 flags, node); 554 flags, node);
569 } 555 }
@@ -608,12 +594,6 @@ void kmem_cache_free(struct kmem_cache *c, void *b)
608} 594}
609EXPORT_SYMBOL(kmem_cache_free); 595EXPORT_SYMBOL(kmem_cache_free);
610 596
611unsigned int kmem_cache_size(struct kmem_cache *c)
612{
613 return c->size;
614}
615EXPORT_SYMBOL(kmem_cache_size);
616
617int __kmem_cache_shutdown(struct kmem_cache *c) 597int __kmem_cache_shutdown(struct kmem_cache *c)
618{ 598{
619 /* No way to check for remaining objects */ 599 /* No way to check for remaining objects */
diff --git a/mm/slub.c b/mm/slub.c
index 487f0bdd53c..87f9f32bf0c 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -112,9 +112,6 @@
112 * the fast path and disables lockless freelists. 112 * the fast path and disables lockless freelists.
113 */ 113 */
114 114
115#define SLAB_DEBUG_FLAGS (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER | \
116 SLAB_TRACE | SLAB_DEBUG_FREE)
117
118static inline int kmem_cache_debug(struct kmem_cache *s) 115static inline int kmem_cache_debug(struct kmem_cache *s)
119{ 116{
120#ifdef CONFIG_SLUB_DEBUG 117#ifdef CONFIG_SLUB_DEBUG
@@ -179,8 +176,6 @@ static inline int kmem_cache_debug(struct kmem_cache *s)
179#define __OBJECT_POISON 0x80000000UL /* Poison object */ 176#define __OBJECT_POISON 0x80000000UL /* Poison object */
180#define __CMPXCHG_DOUBLE 0x40000000UL /* Use cmpxchg_double */ 177#define __CMPXCHG_DOUBLE 0x40000000UL /* Use cmpxchg_double */
181 178
182static int kmem_size = sizeof(struct kmem_cache);
183
184#ifdef CONFIG_SMP 179#ifdef CONFIG_SMP
185static struct notifier_block slab_notifier; 180static struct notifier_block slab_notifier;
186#endif 181#endif
@@ -1092,11 +1087,11 @@ static noinline struct kmem_cache_node *free_debug_processing(
1092 if (!check_object(s, page, object, SLUB_RED_ACTIVE)) 1087 if (!check_object(s, page, object, SLUB_RED_ACTIVE))
1093 goto out; 1088 goto out;
1094 1089
1095 if (unlikely(s != page->slab)) { 1090 if (unlikely(s != page->slab_cache)) {
1096 if (!PageSlab(page)) { 1091 if (!PageSlab(page)) {
1097 slab_err(s, page, "Attempt to free object(0x%p) " 1092 slab_err(s, page, "Attempt to free object(0x%p) "
1098 "outside of slab", object); 1093 "outside of slab", object);
1099 } else if (!page->slab) { 1094 } else if (!page->slab_cache) {
1100 printk(KERN_ERR 1095 printk(KERN_ERR
1101 "SLUB <none>: no slab for object 0x%p.\n", 1096 "SLUB <none>: no slab for object 0x%p.\n",
1102 object); 1097 object);
@@ -1357,7 +1352,7 @@ static struct page *new_slab(struct kmem_cache *s, gfp_t flags, int node)
1357 goto out; 1352 goto out;
1358 1353
1359 inc_slabs_node(s, page_to_nid(page), page->objects); 1354 inc_slabs_node(s, page_to_nid(page), page->objects);
1360 page->slab = s; 1355 page->slab_cache = s;
1361 __SetPageSlab(page); 1356 __SetPageSlab(page);
1362 if (page->pfmemalloc) 1357 if (page->pfmemalloc)
1363 SetPageSlabPfmemalloc(page); 1358 SetPageSlabPfmemalloc(page);
@@ -1424,7 +1419,7 @@ static void rcu_free_slab(struct rcu_head *h)
1424 else 1419 else
1425 page = container_of((struct list_head *)h, struct page, lru); 1420 page = container_of((struct list_head *)h, struct page, lru);
1426 1421
1427 __free_slab(page->slab, page); 1422 __free_slab(page->slab_cache, page);
1428} 1423}
1429 1424
1430static void free_slab(struct kmem_cache *s, struct page *page) 1425static void free_slab(struct kmem_cache *s, struct page *page)
@@ -1872,12 +1867,14 @@ redo:
1872/* 1867/*
1873 * Unfreeze all the cpu partial slabs. 1868 * Unfreeze all the cpu partial slabs.
1874 * 1869 *
1875 * This function must be called with interrupt disabled. 1870 * This function must be called with interrupts disabled
1871 * for the cpu using c (or some other guarantee must be there
1872 * to guarantee no concurrent accesses).
1876 */ 1873 */
1877static void unfreeze_partials(struct kmem_cache *s) 1874static void unfreeze_partials(struct kmem_cache *s,
1875 struct kmem_cache_cpu *c)
1878{ 1876{
1879 struct kmem_cache_node *n = NULL, *n2 = NULL; 1877 struct kmem_cache_node *n = NULL, *n2 = NULL;
1880 struct kmem_cache_cpu *c = this_cpu_ptr(s->cpu_slab);
1881 struct page *page, *discard_page = NULL; 1878 struct page *page, *discard_page = NULL;
1882 1879
1883 while ((page = c->partial)) { 1880 while ((page = c->partial)) {
@@ -1963,7 +1960,7 @@ static int put_cpu_partial(struct kmem_cache *s, struct page *page, int drain)
1963 * set to the per node partial list. 1960 * set to the per node partial list.
1964 */ 1961 */
1965 local_irq_save(flags); 1962 local_irq_save(flags);
1966 unfreeze_partials(s); 1963 unfreeze_partials(s, this_cpu_ptr(s->cpu_slab));
1967 local_irq_restore(flags); 1964 local_irq_restore(flags);
1968 oldpage = NULL; 1965 oldpage = NULL;
1969 pobjects = 0; 1966 pobjects = 0;
@@ -2006,7 +2003,7 @@ static inline void __flush_cpu_slab(struct kmem_cache *s, int cpu)
2006 if (c->page) 2003 if (c->page)
2007 flush_slab(s, c); 2004 flush_slab(s, c);
2008 2005
2009 unfreeze_partials(s); 2006 unfreeze_partials(s, c);
2010 } 2007 }
2011} 2008}
2012 2009
@@ -2459,7 +2456,6 @@ static void __slab_free(struct kmem_cache *s, struct page *page,
2459 void *prior; 2456 void *prior;
2460 void **object = (void *)x; 2457 void **object = (void *)x;
2461 int was_frozen; 2458 int was_frozen;
2462 int inuse;
2463 struct page new; 2459 struct page new;
2464 unsigned long counters; 2460 unsigned long counters;
2465 struct kmem_cache_node *n = NULL; 2461 struct kmem_cache_node *n = NULL;
@@ -2472,13 +2468,17 @@ static void __slab_free(struct kmem_cache *s, struct page *page,
2472 return; 2468 return;
2473 2469
2474 do { 2470 do {
2471 if (unlikely(n)) {
2472 spin_unlock_irqrestore(&n->list_lock, flags);
2473 n = NULL;
2474 }
2475 prior = page->freelist; 2475 prior = page->freelist;
2476 counters = page->counters; 2476 counters = page->counters;
2477 set_freepointer(s, object, prior); 2477 set_freepointer(s, object, prior);
2478 new.counters = counters; 2478 new.counters = counters;
2479 was_frozen = new.frozen; 2479 was_frozen = new.frozen;
2480 new.inuse--; 2480 new.inuse--;
2481 if ((!new.inuse || !prior) && !was_frozen && !n) { 2481 if ((!new.inuse || !prior) && !was_frozen) {
2482 2482
2483 if (!kmem_cache_debug(s) && !prior) 2483 if (!kmem_cache_debug(s) && !prior)
2484 2484
@@ -2503,7 +2503,6 @@ static void __slab_free(struct kmem_cache *s, struct page *page,
2503 2503
2504 } 2504 }
2505 } 2505 }
2506 inuse = new.inuse;
2507 2506
2508 } while (!cmpxchg_double_slab(s, page, 2507 } while (!cmpxchg_double_slab(s, page,
2509 prior, counters, 2508 prior, counters,
@@ -2529,25 +2528,17 @@ static void __slab_free(struct kmem_cache *s, struct page *page,
2529 return; 2528 return;
2530 } 2529 }
2531 2530
2531 if (unlikely(!new.inuse && n->nr_partial > s->min_partial))
2532 goto slab_empty;
2533
2532 /* 2534 /*
2533 * was_frozen may have been set after we acquired the list_lock in 2535 * Objects left in the slab. If it was not on the partial list before
2534 * an earlier loop. So we need to check it here again. 2536 * then add it.
2535 */ 2537 */
2536 if (was_frozen) 2538 if (kmem_cache_debug(s) && unlikely(!prior)) {
2537 stat(s, FREE_FROZEN); 2539 remove_full(s, page);
2538 else { 2540 add_partial(n, page, DEACTIVATE_TO_TAIL);
2539 if (unlikely(!inuse && n->nr_partial > s->min_partial)) 2541 stat(s, FREE_ADD_PARTIAL);
2540 goto slab_empty;
2541
2542 /*
2543 * Objects left in the slab. If it was not on the partial list before
2544 * then add it.
2545 */
2546 if (unlikely(!prior)) {
2547 remove_full(s, page);
2548 add_partial(n, page, DEACTIVATE_TO_TAIL);
2549 stat(s, FREE_ADD_PARTIAL);
2550 }
2551 } 2542 }
2552 spin_unlock_irqrestore(&n->list_lock, flags); 2543 spin_unlock_irqrestore(&n->list_lock, flags);
2553 return; 2544 return;
@@ -2623,9 +2614,9 @@ void kmem_cache_free(struct kmem_cache *s, void *x)
2623 2614
2624 page = virt_to_head_page(x); 2615 page = virt_to_head_page(x);
2625 2616
2626 if (kmem_cache_debug(s) && page->slab != s) { 2617 if (kmem_cache_debug(s) && page->slab_cache != s) {
2627 pr_err("kmem_cache_free: Wrong slab cache. %s but object" 2618 pr_err("kmem_cache_free: Wrong slab cache. %s but object"
2628 " is from %s\n", page->slab->name, s->name); 2619 " is from %s\n", page->slab_cache->name, s->name);
2629 WARN_ON_ONCE(1); 2620 WARN_ON_ONCE(1);
2630 return; 2621 return;
2631 } 2622 }
@@ -2769,32 +2760,6 @@ static inline int calculate_order(int size, int reserved)
2769 return -ENOSYS; 2760 return -ENOSYS;
2770} 2761}
2771 2762
2772/*
2773 * Figure out what the alignment of the objects will be.
2774 */
2775static unsigned long calculate_alignment(unsigned long flags,
2776 unsigned long align, unsigned long size)
2777{
2778 /*
2779 * If the user wants hardware cache aligned objects then follow that
2780 * suggestion if the object is sufficiently large.
2781 *
2782 * The hardware cache alignment cannot override the specified
2783 * alignment though. If that is greater then use it.
2784 */
2785 if (flags & SLAB_HWCACHE_ALIGN) {
2786 unsigned long ralign = cache_line_size();
2787 while (size <= ralign / 2)
2788 ralign /= 2;
2789 align = max(align, ralign);
2790 }
2791
2792 if (align < ARCH_SLAB_MINALIGN)
2793 align = ARCH_SLAB_MINALIGN;
2794
2795 return ALIGN(align, sizeof(void *));
2796}
2797
2798static void 2763static void
2799init_kmem_cache_node(struct kmem_cache_node *n) 2764init_kmem_cache_node(struct kmem_cache_node *n)
2800{ 2765{
@@ -2928,7 +2893,6 @@ static int calculate_sizes(struct kmem_cache *s, int forced_order)
2928{ 2893{
2929 unsigned long flags = s->flags; 2894 unsigned long flags = s->flags;
2930 unsigned long size = s->object_size; 2895 unsigned long size = s->object_size;
2931 unsigned long align = s->align;
2932 int order; 2896 int order;
2933 2897
2934 /* 2898 /*
@@ -3000,19 +2964,11 @@ static int calculate_sizes(struct kmem_cache *s, int forced_order)
3000#endif 2964#endif
3001 2965
3002 /* 2966 /*
3003 * Determine the alignment based on various parameters that the
3004 * user specified and the dynamic determination of cache line size
3005 * on bootup.
3006 */
3007 align = calculate_alignment(flags, align, s->object_size);
3008 s->align = align;
3009
3010 /*
3011 * SLUB stores one object immediately after another beginning from 2967 * SLUB stores one object immediately after another beginning from
3012 * offset 0. In order to align the objects we have to simply size 2968 * offset 0. In order to align the objects we have to simply size
3013 * each object to conform to the alignment. 2969 * each object to conform to the alignment.
3014 */ 2970 */
3015 size = ALIGN(size, align); 2971 size = ALIGN(size, s->align);
3016 s->size = size; 2972 s->size = size;
3017 if (forced_order >= 0) 2973 if (forced_order >= 0)
3018 order = forced_order; 2974 order = forced_order;
@@ -3041,7 +2997,6 @@ static int calculate_sizes(struct kmem_cache *s, int forced_order)
3041 s->max = s->oo; 2997 s->max = s->oo;
3042 2998
3043 return !!oo_objects(s->oo); 2999 return !!oo_objects(s->oo);
3044
3045} 3000}
3046 3001
3047static int kmem_cache_open(struct kmem_cache *s, unsigned long flags) 3002static int kmem_cache_open(struct kmem_cache *s, unsigned long flags)
@@ -3127,15 +3082,6 @@ error:
3127 return -EINVAL; 3082 return -EINVAL;
3128} 3083}
3129 3084
3130/*
3131 * Determine the size of a slab object
3132 */
3133unsigned int kmem_cache_size(struct kmem_cache *s)
3134{
3135 return s->object_size;
3136}
3137EXPORT_SYMBOL(kmem_cache_size);
3138
3139static void list_slab_objects(struct kmem_cache *s, struct page *page, 3085static void list_slab_objects(struct kmem_cache *s, struct page *page,
3140 const char *text) 3086 const char *text)
3141{ 3087{
@@ -3261,32 +3207,6 @@ static int __init setup_slub_nomerge(char *str)
3261 3207
3262__setup("slub_nomerge", setup_slub_nomerge); 3208__setup("slub_nomerge", setup_slub_nomerge);
3263 3209
3264static struct kmem_cache *__init create_kmalloc_cache(const char *name,
3265 int size, unsigned int flags)
3266{
3267 struct kmem_cache *s;
3268
3269 s = kmem_cache_zalloc(kmem_cache, GFP_NOWAIT);
3270
3271 s->name = name;
3272 s->size = s->object_size = size;
3273 s->align = ARCH_KMALLOC_MINALIGN;
3274
3275 /*
3276 * This function is called with IRQs disabled during early-boot on
3277 * single CPU so there's no need to take slab_mutex here.
3278 */
3279 if (kmem_cache_open(s, flags))
3280 goto panic;
3281
3282 list_add(&s->list, &slab_caches);
3283 return s;
3284
3285panic:
3286 panic("Creation of kmalloc slab %s size=%d failed.\n", name, size);
3287 return NULL;
3288}
3289
3290/* 3210/*
3291 * Conversion table for small slabs sizes / 8 to the index in the 3211 * Conversion table for small slabs sizes / 8 to the index in the
3292 * kmalloc array. This is necessary for slabs < 192 since we have non power 3212 * kmalloc array. This is necessary for slabs < 192 since we have non power
@@ -3424,7 +3344,7 @@ size_t ksize(const void *object)
3424 return PAGE_SIZE << compound_order(page); 3344 return PAGE_SIZE << compound_order(page);
3425 } 3345 }
3426 3346
3427 return slab_ksize(page->slab); 3347 return slab_ksize(page->slab_cache);
3428} 3348}
3429EXPORT_SYMBOL(ksize); 3349EXPORT_SYMBOL(ksize);
3430 3350
@@ -3449,8 +3369,8 @@ bool verify_mem_not_deleted(const void *x)
3449 } 3369 }
3450 3370
3451 slab_lock(page); 3371 slab_lock(page);
3452 if (on_freelist(page->slab, page, object)) { 3372 if (on_freelist(page->slab_cache, page, object)) {
3453 object_err(page->slab, page, object, "Object is on free-list"); 3373 object_err(page->slab_cache, page, object, "Object is on free-list");
3454 rv = false; 3374 rv = false;
3455 } else { 3375 } else {
3456 rv = true; 3376 rv = true;
@@ -3481,7 +3401,7 @@ void kfree(const void *x)
3481 __free_pages(page, compound_order(page)); 3401 __free_pages(page, compound_order(page));
3482 return; 3402 return;
3483 } 3403 }
3484 slab_free(page->slab, page, object, _RET_IP_); 3404 slab_free(page->slab_cache, page, object, _RET_IP_);
3485} 3405}
3486EXPORT_SYMBOL(kfree); 3406EXPORT_SYMBOL(kfree);
3487 3407
@@ -3676,15 +3596,16 @@ static int slab_memory_callback(struct notifier_block *self,
3676 3596
3677/* 3597/*
3678 * Used for early kmem_cache structures that were allocated using 3598 * Used for early kmem_cache structures that were allocated using
3679 * the page allocator 3599 * the page allocator. Allocate them properly then fix up the pointers
3600 * that may be pointing to the wrong kmem_cache structure.
3680 */ 3601 */
3681 3602
3682static void __init kmem_cache_bootstrap_fixup(struct kmem_cache *s) 3603static struct kmem_cache * __init bootstrap(struct kmem_cache *static_cache)
3683{ 3604{
3684 int node; 3605 int node;
3606 struct kmem_cache *s = kmem_cache_zalloc(kmem_cache, GFP_NOWAIT);
3685 3607
3686 list_add(&s->list, &slab_caches); 3608 memcpy(s, static_cache, kmem_cache->object_size);
3687 s->refcount = -1;
3688 3609
3689 for_each_node_state(node, N_NORMAL_MEMORY) { 3610 for_each_node_state(node, N_NORMAL_MEMORY) {
3690 struct kmem_cache_node *n = get_node(s, node); 3611 struct kmem_cache_node *n = get_node(s, node);
@@ -3692,78 +3613,52 @@ static void __init kmem_cache_bootstrap_fixup(struct kmem_cache *s)
3692 3613
3693 if (n) { 3614 if (n) {
3694 list_for_each_entry(p, &n->partial, lru) 3615 list_for_each_entry(p, &n->partial, lru)
3695 p->slab = s; 3616 p->slab_cache = s;
3696 3617
3697#ifdef CONFIG_SLUB_DEBUG 3618#ifdef CONFIG_SLUB_DEBUG
3698 list_for_each_entry(p, &n->full, lru) 3619 list_for_each_entry(p, &n->full, lru)
3699 p->slab = s; 3620 p->slab_cache = s;
3700#endif 3621#endif
3701 } 3622 }
3702 } 3623 }
3624 list_add(&s->list, &slab_caches);
3625 return s;
3703} 3626}
3704 3627
3705void __init kmem_cache_init(void) 3628void __init kmem_cache_init(void)
3706{ 3629{
3630 static __initdata struct kmem_cache boot_kmem_cache,
3631 boot_kmem_cache_node;
3707 int i; 3632 int i;
3708 int caches = 0; 3633 int caches = 2;
3709 struct kmem_cache *temp_kmem_cache;
3710 int order;
3711 struct kmem_cache *temp_kmem_cache_node;
3712 unsigned long kmalloc_size;
3713 3634
3714 if (debug_guardpage_minorder()) 3635 if (debug_guardpage_minorder())
3715 slub_max_order = 0; 3636 slub_max_order = 0;
3716 3637
3717 kmem_size = offsetof(struct kmem_cache, node) + 3638 kmem_cache_node = &boot_kmem_cache_node;
3718 nr_node_ids * sizeof(struct kmem_cache_node *); 3639 kmem_cache = &boot_kmem_cache;
3719
3720 /* Allocate two kmem_caches from the page allocator */
3721 kmalloc_size = ALIGN(kmem_size, cache_line_size());
3722 order = get_order(2 * kmalloc_size);
3723 kmem_cache = (void *)__get_free_pages(GFP_NOWAIT | __GFP_ZERO, order);
3724
3725 /*
3726 * Must first have the slab cache available for the allocations of the
3727 * struct kmem_cache_node's. There is special bootstrap code in
3728 * kmem_cache_open for slab_state == DOWN.
3729 */
3730 kmem_cache_node = (void *)kmem_cache + kmalloc_size;
3731 3640
3732 kmem_cache_node->name = "kmem_cache_node"; 3641 create_boot_cache(kmem_cache_node, "kmem_cache_node",
3733 kmem_cache_node->size = kmem_cache_node->object_size = 3642 sizeof(struct kmem_cache_node), SLAB_HWCACHE_ALIGN);
3734 sizeof(struct kmem_cache_node);
3735 kmem_cache_open(kmem_cache_node, SLAB_HWCACHE_ALIGN | SLAB_PANIC);
3736 3643
3737 hotplug_memory_notifier(slab_memory_callback, SLAB_CALLBACK_PRI); 3644 hotplug_memory_notifier(slab_memory_callback, SLAB_CALLBACK_PRI);
3738 3645
3739 /* Able to allocate the per node structures */ 3646 /* Able to allocate the per node structures */
3740 slab_state = PARTIAL; 3647 slab_state = PARTIAL;
3741 3648
3742 temp_kmem_cache = kmem_cache; 3649 create_boot_cache(kmem_cache, "kmem_cache",
3743 kmem_cache->name = "kmem_cache"; 3650 offsetof(struct kmem_cache, node) +
3744 kmem_cache->size = kmem_cache->object_size = kmem_size; 3651 nr_node_ids * sizeof(struct kmem_cache_node *),
3745 kmem_cache_open(kmem_cache, SLAB_HWCACHE_ALIGN | SLAB_PANIC); 3652 SLAB_HWCACHE_ALIGN);
3746 3653
3747 kmem_cache = kmem_cache_alloc(kmem_cache, GFP_NOWAIT); 3654 kmem_cache = bootstrap(&boot_kmem_cache);
3748 memcpy(kmem_cache, temp_kmem_cache, kmem_size);
3749 3655
3750 /* 3656 /*
3751 * Allocate kmem_cache_node properly from the kmem_cache slab. 3657 * Allocate kmem_cache_node properly from the kmem_cache slab.
3752 * kmem_cache_node is separately allocated so no need to 3658 * kmem_cache_node is separately allocated so no need to
3753 * update any list pointers. 3659 * update any list pointers.
3754 */ 3660 */
3755 temp_kmem_cache_node = kmem_cache_node; 3661 kmem_cache_node = bootstrap(&boot_kmem_cache_node);
3756
3757 kmem_cache_node = kmem_cache_alloc(kmem_cache, GFP_NOWAIT);
3758 memcpy(kmem_cache_node, temp_kmem_cache_node, kmem_size);
3759
3760 kmem_cache_bootstrap_fixup(kmem_cache_node);
3761
3762 caches++;
3763 kmem_cache_bootstrap_fixup(kmem_cache);
3764 caches++;
3765 /* Free temporary boot structure */
3766 free_pages((unsigned long)temp_kmem_cache, order);
3767 3662
3768 /* Now we can use the kmem_cache to allocate kmalloc slabs */ 3663 /* Now we can use the kmem_cache to allocate kmalloc slabs */
3769 3664
@@ -3964,6 +3859,10 @@ int __kmem_cache_create(struct kmem_cache *s, unsigned long flags)
3964 if (err) 3859 if (err)
3965 return err; 3860 return err;
3966 3861
3862 /* Mutex is not taken during early boot */
3863 if (slab_state <= UP)
3864 return 0;
3865
3967 mutex_unlock(&slab_mutex); 3866 mutex_unlock(&slab_mutex);
3968 err = sysfs_slab_add(s); 3867 err = sysfs_slab_add(s);
3969 mutex_lock(&slab_mutex); 3868 mutex_lock(&slab_mutex);
@@ -5265,13 +5164,8 @@ static int sysfs_slab_add(struct kmem_cache *s)
5265{ 5164{
5266 int err; 5165 int err;
5267 const char *name; 5166 const char *name;
5268 int unmergeable; 5167 int unmergeable = slab_unmergeable(s);
5269 5168
5270 if (slab_state < FULL)
5271 /* Defer until later */
5272 return 0;
5273
5274 unmergeable = slab_unmergeable(s);
5275 if (unmergeable) { 5169 if (unmergeable) {
5276 /* 5170 /*
5277 * Slabcache can never be merged so we can use the name proper. 5171 * Slabcache can never be merged so we can use the name proper.
@@ -5405,49 +5299,14 @@ __initcall(slab_sysfs_init);
5405 * The /proc/slabinfo ABI 5299 * The /proc/slabinfo ABI
5406 */ 5300 */
5407#ifdef CONFIG_SLABINFO 5301#ifdef CONFIG_SLABINFO
5408static void print_slabinfo_header(struct seq_file *m) 5302void get_slabinfo(struct kmem_cache *s, struct slabinfo *sinfo)
5409{
5410 seq_puts(m, "slabinfo - version: 2.1\n");
5411 seq_puts(m, "# name <active_objs> <num_objs> <object_size> "
5412 "<objperslab> <pagesperslab>");
5413 seq_puts(m, " : tunables <limit> <batchcount> <sharedfactor>");
5414 seq_puts(m, " : slabdata <active_slabs> <num_slabs> <sharedavail>");
5415 seq_putc(m, '\n');
5416}
5417
5418static void *s_start(struct seq_file *m, loff_t *pos)
5419{
5420 loff_t n = *pos;
5421
5422 mutex_lock(&slab_mutex);
5423 if (!n)
5424 print_slabinfo_header(m);
5425
5426 return seq_list_start(&slab_caches, *pos);
5427}
5428
5429static void *s_next(struct seq_file *m, void *p, loff_t *pos)
5430{
5431 return seq_list_next(p, &slab_caches, pos);
5432}
5433
5434static void s_stop(struct seq_file *m, void *p)
5435{
5436 mutex_unlock(&slab_mutex);
5437}
5438
5439static int s_show(struct seq_file *m, void *p)
5440{ 5303{
5441 unsigned long nr_partials = 0; 5304 unsigned long nr_partials = 0;
5442 unsigned long nr_slabs = 0; 5305 unsigned long nr_slabs = 0;
5443 unsigned long nr_inuse = 0;
5444 unsigned long nr_objs = 0; 5306 unsigned long nr_objs = 0;
5445 unsigned long nr_free = 0; 5307 unsigned long nr_free = 0;
5446 struct kmem_cache *s;
5447 int node; 5308 int node;
5448 5309
5449 s = list_entry(p, struct kmem_cache, list);
5450
5451 for_each_online_node(node) { 5310 for_each_online_node(node) {
5452 struct kmem_cache_node *n = get_node(s, node); 5311 struct kmem_cache_node *n = get_node(s, node);
5453 5312
@@ -5460,41 +5319,21 @@ static int s_show(struct seq_file *m, void *p)
5460 nr_free += count_partial(n, count_free); 5319 nr_free += count_partial(n, count_free);
5461 } 5320 }
5462 5321
5463 nr_inuse = nr_objs - nr_free; 5322 sinfo->active_objs = nr_objs - nr_free;
5464 5323 sinfo->num_objs = nr_objs;
5465 seq_printf(m, "%-17s %6lu %6lu %6u %4u %4d", s->name, nr_inuse, 5324 sinfo->active_slabs = nr_slabs;
5466 nr_objs, s->size, oo_objects(s->oo), 5325 sinfo->num_slabs = nr_slabs;
5467 (1 << oo_order(s->oo))); 5326 sinfo->objects_per_slab = oo_objects(s->oo);
5468 seq_printf(m, " : tunables %4u %4u %4u", 0, 0, 0); 5327 sinfo->cache_order = oo_order(s->oo);
5469 seq_printf(m, " : slabdata %6lu %6lu %6lu", nr_slabs, nr_slabs,
5470 0UL);
5471 seq_putc(m, '\n');
5472 return 0;
5473} 5328}
5474 5329
5475static const struct seq_operations slabinfo_op = { 5330void slabinfo_show_stats(struct seq_file *m, struct kmem_cache *s)
5476 .start = s_start,
5477 .next = s_next,
5478 .stop = s_stop,
5479 .show = s_show,
5480};
5481
5482static int slabinfo_open(struct inode *inode, struct file *file)
5483{ 5331{
5484 return seq_open(file, &slabinfo_op);
5485} 5332}
5486 5333
5487static const struct file_operations proc_slabinfo_operations = { 5334ssize_t slabinfo_write(struct file *file, const char __user *buffer,
5488 .open = slabinfo_open, 5335 size_t count, loff_t *ppos)
5489 .read = seq_read,
5490 .llseek = seq_lseek,
5491 .release = seq_release,
5492};
5493
5494static int __init slab_proc_init(void)
5495{ 5336{
5496 proc_create("slabinfo", S_IRUSR, NULL, &proc_slabinfo_operations); 5337 return -EIO;
5497 return 0;
5498} 5338}
5499module_init(slab_proc_init);
5500#endif /* CONFIG_SLABINFO */ 5339#endif /* CONFIG_SLABINFO */
generated by cgit v1.2.2 (git 2.25.0) at 2025-03-25 02:58:33 -0400