aboutsummaryrefslogtreecommitdiffstats
path: root/kernel
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2010-04-05 12:16:37 -0400
committerLinus Torvalds <torvalds@linux-foundation.org>2010-04-05 12:16:37 -0400
commit9e74e7c81a24aee66024fc477786bd1de84e293b (patch)
tree8f172b7e5a86fe6b51e46f1d219e9c0707beab60 /kernel
parent4946d54cb55e86a156216fcfeed5568514b0830f (diff)
parentd5e50daf92df8afcb701fd717b301985691e802f (diff)
Merge branch 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/percpu
* 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/percpu: module: add stub for is_module_percpu_address percpu, module: implement and use is_kernel/module_percpu_address() module: encapsulate percpu handling better and record percpu_size
Diffstat (limited to 'kernel')
-rw-r--r--kernel/lockdep.c21
-rw-r--r--kernel/module.c104
2 files changed, 78 insertions, 47 deletions
diff --git a/kernel/lockdep.c b/kernel/lockdep.c
index c927a549db2c..9bbb9c841e48 100644
--- a/kernel/lockdep.c
+++ b/kernel/lockdep.c
@@ -582,9 +582,6 @@ static int static_obj(void *obj)
582 unsigned long start = (unsigned long) &_stext, 582 unsigned long start = (unsigned long) &_stext,
583 end = (unsigned long) &_end, 583 end = (unsigned long) &_end,
584 addr = (unsigned long) obj; 584 addr = (unsigned long) obj;
585#ifdef CONFIG_SMP
586 int i;
587#endif
588 585
589 /* 586 /*
590 * static variable? 587 * static variable?
@@ -595,24 +592,16 @@ static int static_obj(void *obj)
595 if (arch_is_kernel_data(addr)) 592 if (arch_is_kernel_data(addr))
596 return 1; 593 return 1;
597 594
598#ifdef CONFIG_SMP
599 /* 595 /*
600 * percpu var? 596 * in-kernel percpu var?
601 */ 597 */
602 for_each_possible_cpu(i) { 598 if (is_kernel_percpu_address(addr))
603 start = (unsigned long) &__per_cpu_start + per_cpu_offset(i); 599 return 1;
604 end = (unsigned long) &__per_cpu_start + PERCPU_ENOUGH_ROOM
605 + per_cpu_offset(i);
606
607 if ((addr >= start) && (addr < end))
608 return 1;
609 }
610#endif
611 600
612 /* 601 /*
613 * module var? 602 * module static or percpu var?
614 */ 603 */
615 return is_module_address(addr); 604 return is_module_address(addr) || is_module_percpu_address(addr);
616} 605}
617 606
618/* 607/*
diff --git a/kernel/module.c b/kernel/module.c
index c968d3606dca..9f8d23d8b3a8 100644
--- a/kernel/module.c
+++ b/kernel/module.c
@@ -370,27 +370,33 @@ EXPORT_SYMBOL_GPL(find_module);
370 370
371#ifdef CONFIG_SMP 371#ifdef CONFIG_SMP
372 372
373static void *percpu_modalloc(unsigned long size, unsigned long align, 373static inline void __percpu *mod_percpu(struct module *mod)
374 const char *name)
375{ 374{
376 void *ptr; 375 return mod->percpu;
376}
377 377
378static int percpu_modalloc(struct module *mod,
379 unsigned long size, unsigned long align)
380{
378 if (align > PAGE_SIZE) { 381 if (align > PAGE_SIZE) {
379 printk(KERN_WARNING "%s: per-cpu alignment %li > %li\n", 382 printk(KERN_WARNING "%s: per-cpu alignment %li > %li\n",
380 name, align, PAGE_SIZE); 383 mod->name, align, PAGE_SIZE);
381 align = PAGE_SIZE; 384 align = PAGE_SIZE;
382 } 385 }
383 386
384 ptr = __alloc_reserved_percpu(size, align); 387 mod->percpu = __alloc_reserved_percpu(size, align);
385 if (!ptr) 388 if (!mod->percpu) {
386 printk(KERN_WARNING 389 printk(KERN_WARNING
387 "Could not allocate %lu bytes percpu data\n", size); 390 "Could not allocate %lu bytes percpu data\n", size);
388 return ptr; 391 return -ENOMEM;
392 }
393 mod->percpu_size = size;
394 return 0;
389} 395}
390 396
391static void percpu_modfree(void *freeme) 397static void percpu_modfree(struct module *mod)
392{ 398{
393 free_percpu(freeme); 399 free_percpu(mod->percpu);
394} 400}
395 401
396static unsigned int find_pcpusec(Elf_Ehdr *hdr, 402static unsigned int find_pcpusec(Elf_Ehdr *hdr,
@@ -400,24 +406,62 @@ static unsigned int find_pcpusec(Elf_Ehdr *hdr,
400 return find_sec(hdr, sechdrs, secstrings, ".data.percpu"); 406 return find_sec(hdr, sechdrs, secstrings, ".data.percpu");
401} 407}
402 408
403static void percpu_modcopy(void *pcpudest, const void *from, unsigned long size) 409static void percpu_modcopy(struct module *mod,
410 const void *from, unsigned long size)
404{ 411{
405 int cpu; 412 int cpu;
406 413
407 for_each_possible_cpu(cpu) 414 for_each_possible_cpu(cpu)
408 memcpy(pcpudest + per_cpu_offset(cpu), from, size); 415 memcpy(per_cpu_ptr(mod->percpu, cpu), from, size);
416}
417
418/**
419 * is_module_percpu_address - test whether address is from module static percpu
420 * @addr: address to test
421 *
422 * Test whether @addr belongs to module static percpu area.
423 *
424 * RETURNS:
425 * %true if @addr is from module static percpu area
426 */
427bool is_module_percpu_address(unsigned long addr)
428{
429 struct module *mod;
430 unsigned int cpu;
431
432 preempt_disable();
433
434 list_for_each_entry_rcu(mod, &modules, list) {
435 if (!mod->percpu_size)
436 continue;
437 for_each_possible_cpu(cpu) {
438 void *start = per_cpu_ptr(mod->percpu, cpu);
439
440 if ((void *)addr >= start &&
441 (void *)addr < start + mod->percpu_size) {
442 preempt_enable();
443 return true;
444 }
445 }
446 }
447
448 preempt_enable();
449 return false;
409} 450}
410 451
411#else /* ... !CONFIG_SMP */ 452#else /* ... !CONFIG_SMP */
412 453
413static inline void *percpu_modalloc(unsigned long size, unsigned long align, 454static inline void __percpu *mod_percpu(struct module *mod)
414 const char *name)
415{ 455{
416 return NULL; 456 return NULL;
417} 457}
418static inline void percpu_modfree(void *pcpuptr) 458static inline int percpu_modalloc(struct module *mod,
459 unsigned long size, unsigned long align)
460{
461 return -ENOMEM;
462}
463static inline void percpu_modfree(struct module *mod)
419{ 464{
420 BUG();
421} 465}
422static inline unsigned int find_pcpusec(Elf_Ehdr *hdr, 466static inline unsigned int find_pcpusec(Elf_Ehdr *hdr,
423 Elf_Shdr *sechdrs, 467 Elf_Shdr *sechdrs,
@@ -425,12 +469,16 @@ static inline unsigned int find_pcpusec(Elf_Ehdr *hdr,
425{ 469{
426 return 0; 470 return 0;
427} 471}
428static inline void percpu_modcopy(void *pcpudst, const void *src, 472static inline void percpu_modcopy(struct module *mod,
429 unsigned long size) 473 const void *from, unsigned long size)
430{ 474{
431 /* pcpusec should be 0, and size of that section should be 0. */ 475 /* pcpusec should be 0, and size of that section should be 0. */
432 BUG_ON(size != 0); 476 BUG_ON(size != 0);
433} 477}
478bool is_module_percpu_address(unsigned long addr)
479{
480 return false;
481}
434 482
435#endif /* CONFIG_SMP */ 483#endif /* CONFIG_SMP */
436 484
@@ -1400,8 +1448,7 @@ static void free_module(struct module *mod)
1400 /* This may be NULL, but that's OK */ 1448 /* This may be NULL, but that's OK */
1401 module_free(mod, mod->module_init); 1449 module_free(mod, mod->module_init);
1402 kfree(mod->args); 1450 kfree(mod->args);
1403 if (mod->percpu) 1451 percpu_modfree(mod);
1404 percpu_modfree(mod->percpu);
1405#if defined(CONFIG_MODULE_UNLOAD) 1452#if defined(CONFIG_MODULE_UNLOAD)
1406 if (mod->refptr) 1453 if (mod->refptr)
1407 free_percpu(mod->refptr); 1454 free_percpu(mod->refptr);
@@ -1520,7 +1567,7 @@ static int simplify_symbols(Elf_Shdr *sechdrs,
1520 default: 1567 default:
1521 /* Divert to percpu allocation if a percpu var. */ 1568 /* Divert to percpu allocation if a percpu var. */
1522 if (sym[i].st_shndx == pcpuindex) 1569 if (sym[i].st_shndx == pcpuindex)
1523 secbase = (unsigned long)mod->percpu; 1570 secbase = (unsigned long)mod_percpu(mod);
1524 else 1571 else
1525 secbase = sechdrs[sym[i].st_shndx].sh_addr; 1572 secbase = sechdrs[sym[i].st_shndx].sh_addr;
1526 sym[i].st_value += secbase; 1573 sym[i].st_value += secbase;
@@ -1954,7 +2001,7 @@ static noinline struct module *load_module(void __user *umod,
1954 unsigned int modindex, versindex, infoindex, pcpuindex; 2001 unsigned int modindex, versindex, infoindex, pcpuindex;
1955 struct module *mod; 2002 struct module *mod;
1956 long err = 0; 2003 long err = 0;
1957 void *percpu = NULL, *ptr = NULL; /* Stops spurious gcc warning */ 2004 void *ptr = NULL; /* Stops spurious gcc warning */
1958 unsigned long symoffs, stroffs, *strmap; 2005 unsigned long symoffs, stroffs, *strmap;
1959 2006
1960 mm_segment_t old_fs; 2007 mm_segment_t old_fs;
@@ -2094,15 +2141,11 @@ static noinline struct module *load_module(void __user *umod,
2094 2141
2095 if (pcpuindex) { 2142 if (pcpuindex) {
2096 /* We have a special allocation for this section. */ 2143 /* We have a special allocation for this section. */
2097 percpu = percpu_modalloc(sechdrs[pcpuindex].sh_size, 2144 err = percpu_modalloc(mod, sechdrs[pcpuindex].sh_size,
2098 sechdrs[pcpuindex].sh_addralign, 2145 sechdrs[pcpuindex].sh_addralign);
2099 mod->name); 2146 if (err)
2100 if (!percpu) {
2101 err = -ENOMEM;
2102 goto free_mod; 2147 goto free_mod;
2103 }
2104 sechdrs[pcpuindex].sh_flags &= ~(unsigned long)SHF_ALLOC; 2148 sechdrs[pcpuindex].sh_flags &= ~(unsigned long)SHF_ALLOC;
2105 mod->percpu = percpu;
2106 } 2149 }
2107 2150
2108 /* Determine total sizes, and put offsets in sh_entsize. For now 2151 /* Determine total sizes, and put offsets in sh_entsize. For now
@@ -2317,7 +2360,7 @@ static noinline struct module *load_module(void __user *umod,
2317 sort_extable(mod->extable, mod->extable + mod->num_exentries); 2360 sort_extable(mod->extable, mod->extable + mod->num_exentries);
2318 2361
2319 /* Finally, copy percpu area over. */ 2362 /* Finally, copy percpu area over. */
2320 percpu_modcopy(mod->percpu, (void *)sechdrs[pcpuindex].sh_addr, 2363 percpu_modcopy(mod, (void *)sechdrs[pcpuindex].sh_addr,
2321 sechdrs[pcpuindex].sh_size); 2364 sechdrs[pcpuindex].sh_size);
2322 2365
2323 add_kallsyms(mod, sechdrs, hdr->e_shnum, symindex, strindex, 2366 add_kallsyms(mod, sechdrs, hdr->e_shnum, symindex, strindex,
@@ -2409,8 +2452,7 @@ static noinline struct module *load_module(void __user *umod,
2409 module_free(mod, mod->module_core); 2452 module_free(mod, mod->module_core);
2410 /* mod will be freed with core. Don't access it beyond this line! */ 2453 /* mod will be freed with core. Don't access it beyond this line! */
2411 free_percpu: 2454 free_percpu:
2412 if (percpu) 2455 percpu_modfree(mod);
2413 percpu_modfree(percpu);
2414 free_mod: 2456 free_mod:
2415 kfree(args); 2457 kfree(args);
2416 kfree(strmap); 2458 kfree(strmap);