diff options
author | Eric Dumazet <dada1@cosmosbay.com> | 2009-02-02 22:01:36 -0500 |
---|---|---|
committer | Linus Torvalds <torvalds@linux-foundation.org> | 2009-02-02 22:17:55 -0500 |
commit | 720eba31f47aeade8ec130ca7f4353223c49170f (patch) | |
tree | 4aef103b59a0fb7b9406fd3e4bb09bdf7e3ad2f7 /include/linux/module.h | |
parent | 27421e211a39784694b597dbf35848b88363c248 (diff) |
modules: Use a better scheme for refcounting
Current refcounting for modules (done if CONFIG_MODULE_UNLOAD=y) is
using a lot of memory.
Each 'struct module' contains an [NR_CPUS] array of full cache lines.
This patch uses existing infrastructure (percpu_modalloc() &
percpu_modfree()) to allocate percpu space for the refcount storage.
Instead of wasting NR_CPUS*128 bytes (on i386), we now use
nr_cpu_ids*sizeof(local_t) bytes.
On a typical distro, where NR_CPUS=8, shiping 2000 modules, we reduce
size of module files by about 2 Mbytes. (1Kb per module)
Instead of having all refcounters in the same memory node - with TLB misses
because of vmalloc() - this new implementation permits to have better
NUMA properties, since each CPU will use storage on its preferred node,
thanks to percpu storage.
Signed-off-by: Eric Dumazet <dada1@cosmosbay.com>
Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
Diffstat (limited to 'include/linux/module.h')
-rw-r--r-- | include/linux/module.h | 25 |
1 files changed, 16 insertions, 9 deletions
diff --git a/include/linux/module.h b/include/linux/module.h index 4f7ea12463d..f3b8329eb5b 100644 --- a/include/linux/module.h +++ b/include/linux/module.h | |||
@@ -219,11 +219,6 @@ void *__symbol_get_gpl(const char *symbol); | |||
219 | 219 | ||
220 | #endif | 220 | #endif |
221 | 221 | ||
222 | struct module_ref | ||
223 | { | ||
224 | local_t count; | ||
225 | } ____cacheline_aligned; | ||
226 | |||
227 | enum module_state | 222 | enum module_state |
228 | { | 223 | { |
229 | MODULE_STATE_LIVE, | 224 | MODULE_STATE_LIVE, |
@@ -344,8 +339,11 @@ struct module | |||
344 | /* Destruction function. */ | 339 | /* Destruction function. */ |
345 | void (*exit)(void); | 340 | void (*exit)(void); |
346 | 341 | ||
347 | /* Reference counts */ | 342 | #ifdef CONFIG_SMP |
348 | struct module_ref ref[NR_CPUS]; | 343 | char *refptr; |
344 | #else | ||
345 | local_t ref; | ||
346 | #endif | ||
349 | #endif | 347 | #endif |
350 | }; | 348 | }; |
351 | #ifndef MODULE_ARCH_INIT | 349 | #ifndef MODULE_ARCH_INIT |
@@ -395,13 +393,22 @@ void __symbol_put(const char *symbol); | |||
395 | #define symbol_put(x) __symbol_put(MODULE_SYMBOL_PREFIX #x) | 393 | #define symbol_put(x) __symbol_put(MODULE_SYMBOL_PREFIX #x) |
396 | void symbol_put_addr(void *addr); | 394 | void symbol_put_addr(void *addr); |
397 | 395 | ||
396 | static inline local_t *__module_ref_addr(struct module *mod, int cpu) | ||
397 | { | ||
398 | #ifdef CONFIG_SMP | ||
399 | return (local_t *) (mod->refptr + per_cpu_offset(cpu)); | ||
400 | #else | ||
401 | return &mod->ref; | ||
402 | #endif | ||
403 | } | ||
404 | |||
398 | /* Sometimes we know we already have a refcount, and it's easier not | 405 | /* Sometimes we know we already have a refcount, and it's easier not |
399 | to handle the error case (which only happens with rmmod --wait). */ | 406 | to handle the error case (which only happens with rmmod --wait). */ |
400 | static inline void __module_get(struct module *module) | 407 | static inline void __module_get(struct module *module) |
401 | { | 408 | { |
402 | if (module) { | 409 | if (module) { |
403 | BUG_ON(module_refcount(module) == 0); | 410 | BUG_ON(module_refcount(module) == 0); |
404 | local_inc(&module->ref[get_cpu()].count); | 411 | local_inc(__module_ref_addr(module, get_cpu())); |
405 | put_cpu(); | 412 | put_cpu(); |
406 | } | 413 | } |
407 | } | 414 | } |
@@ -413,7 +420,7 @@ static inline int try_module_get(struct module *module) | |||
413 | if (module) { | 420 | if (module) { |
414 | unsigned int cpu = get_cpu(); | 421 | unsigned int cpu = get_cpu(); |
415 | if (likely(module_is_live(module))) | 422 | if (likely(module_is_live(module))) |
416 | local_inc(&module->ref[cpu].count); | 423 | local_inc(__module_ref_addr(module, cpu)); |
417 | else | 424 | else |
418 | ret = 0; | 425 | ret = 0; |
419 | put_cpu(); | 426 | put_cpu(); |