diff options
-rw-r--r-- | include/linux/module.h | 1 | ||||
-rw-r--r-- | include/linux/percpu.h | 7 | ||||
-rw-r--r-- | kernel/lockdep.c | 21 | ||||
-rw-r--r-- | kernel/module.c | 38 | ||||
-rw-r--r-- | mm/percpu.c | 26 |
5 files changed, 77 insertions, 16 deletions
diff --git a/include/linux/module.h b/include/linux/module.h index 87d247ac6761..f0e2659f4e3e 100644 --- a/include/linux/module.h +++ b/include/linux/module.h | |||
@@ -395,6 +395,7 @@ static inline int module_is_live(struct module *mod) | |||
395 | struct module *__module_text_address(unsigned long addr); | 395 | struct module *__module_text_address(unsigned long addr); |
396 | struct module *__module_address(unsigned long addr); | 396 | struct module *__module_address(unsigned long addr); |
397 | bool is_module_address(unsigned long addr); | 397 | bool is_module_address(unsigned long addr); |
398 | bool is_module_percpu_address(unsigned long addr); | ||
398 | bool is_module_text_address(unsigned long addr); | 399 | bool is_module_text_address(unsigned long addr); |
399 | 400 | ||
400 | static inline int within_module_core(unsigned long addr, struct module *mod) | 401 | static inline int within_module_core(unsigned long addr, struct module *mod) |
diff --git a/include/linux/percpu.h b/include/linux/percpu.h index a93e5bfdccb8..11d5f834b54a 100644 --- a/include/linux/percpu.h +++ b/include/linux/percpu.h | |||
@@ -137,6 +137,7 @@ extern int __init pcpu_page_first_chunk(size_t reserved_size, | |||
137 | extern void __percpu *__alloc_reserved_percpu(size_t size, size_t align); | 137 | extern void __percpu *__alloc_reserved_percpu(size_t size, size_t align); |
138 | extern void __percpu *__alloc_percpu(size_t size, size_t align); | 138 | extern void __percpu *__alloc_percpu(size_t size, size_t align); |
139 | extern void free_percpu(void __percpu *__pdata); | 139 | extern void free_percpu(void __percpu *__pdata); |
140 | extern bool is_kernel_percpu_address(unsigned long addr); | ||
140 | extern phys_addr_t per_cpu_ptr_to_phys(void *addr); | 141 | extern phys_addr_t per_cpu_ptr_to_phys(void *addr); |
141 | 142 | ||
142 | #ifndef CONFIG_HAVE_SETUP_PER_CPU_AREA | 143 | #ifndef CONFIG_HAVE_SETUP_PER_CPU_AREA |
@@ -163,6 +164,12 @@ static inline void free_percpu(void __percpu *p) | |||
163 | kfree(p); | 164 | kfree(p); |
164 | } | 165 | } |
165 | 166 | ||
167 | /* can't distinguish from other static vars, always false */ | ||
168 | static inline bool is_kernel_percpu_address(unsigned long addr) | ||
169 | { | ||
170 | return false; | ||
171 | } | ||
172 | |||
166 | static inline phys_addr_t per_cpu_ptr_to_phys(void *addr) | 173 | static inline phys_addr_t per_cpu_ptr_to_phys(void *addr) |
167 | { | 174 | { |
168 | return __pa(addr); | 175 | return __pa(addr); |
diff --git a/kernel/lockdep.c b/kernel/lockdep.c index c927a549db2c..9bbb9c841e48 100644 --- a/kernel/lockdep.c +++ b/kernel/lockdep.c | |||
@@ -582,9 +582,6 @@ static int static_obj(void *obj) | |||
582 | unsigned long start = (unsigned long) &_stext, | 582 | unsigned long start = (unsigned long) &_stext, |
583 | end = (unsigned long) &_end, | 583 | end = (unsigned long) &_end, |
584 | addr = (unsigned long) obj; | 584 | addr = (unsigned long) obj; |
585 | #ifdef CONFIG_SMP | ||
586 | int i; | ||
587 | #endif | ||
588 | 585 | ||
589 | /* | 586 | /* |
590 | * static variable? | 587 | * static variable? |
@@ -595,24 +592,16 @@ static int static_obj(void *obj) | |||
595 | if (arch_is_kernel_data(addr)) | 592 | if (arch_is_kernel_data(addr)) |
596 | return 1; | 593 | return 1; |
597 | 594 | ||
598 | #ifdef CONFIG_SMP | ||
599 | /* | 595 | /* |
600 | * percpu var? | 596 | * in-kernel percpu var? |
601 | */ | 597 | */ |
602 | for_each_possible_cpu(i) { | 598 | if (is_kernel_percpu_address(addr)) |
603 | start = (unsigned long) &__per_cpu_start + per_cpu_offset(i); | 599 | return 1; |
604 | end = (unsigned long) &__per_cpu_start + PERCPU_ENOUGH_ROOM | ||
605 | + per_cpu_offset(i); | ||
606 | |||
607 | if ((addr >= start) && (addr < end)) | ||
608 | return 1; | ||
609 | } | ||
610 | #endif | ||
611 | 600 | ||
612 | /* | 601 | /* |
613 | * module var? | 602 | * module static or percpu var? |
614 | */ | 603 | */ |
615 | return is_module_address(addr); | 604 | return is_module_address(addr) || is_module_percpu_address(addr); |
616 | } | 605 | } |
617 | 606 | ||
618 | /* | 607 | /* |
diff --git a/kernel/module.c b/kernel/module.c index e7a6e53fc73e..9f8d23d8b3a8 100644 --- a/kernel/module.c +++ b/kernel/module.c | |||
@@ -415,6 +415,40 @@ static void percpu_modcopy(struct module *mod, | |||
415 | memcpy(per_cpu_ptr(mod->percpu, cpu), from, size); | 415 | memcpy(per_cpu_ptr(mod->percpu, cpu), from, size); |
416 | } | 416 | } |
417 | 417 | ||
418 | /** | ||
419 | * is_module_percpu_address - test whether address is from module static percpu | ||
420 | * @addr: address to test | ||
421 | * | ||
422 | * Test whether @addr belongs to module static percpu area. | ||
423 | * | ||
424 | * RETURNS: | ||
425 | * %true if @addr is from module static percpu area | ||
426 | */ | ||
427 | bool is_module_percpu_address(unsigned long addr) | ||
428 | { | ||
429 | struct module *mod; | ||
430 | unsigned int cpu; | ||
431 | |||
432 | preempt_disable(); | ||
433 | |||
434 | list_for_each_entry_rcu(mod, &modules, list) { | ||
435 | if (!mod->percpu_size) | ||
436 | continue; | ||
437 | for_each_possible_cpu(cpu) { | ||
438 | void *start = per_cpu_ptr(mod->percpu, cpu); | ||
439 | |||
440 | if ((void *)addr >= start && | ||
441 | (void *)addr < start + mod->percpu_size) { | ||
442 | preempt_enable(); | ||
443 | return true; | ||
444 | } | ||
445 | } | ||
446 | } | ||
447 | |||
448 | preempt_enable(); | ||
449 | return false; | ||
450 | } | ||
451 | |||
418 | #else /* ... !CONFIG_SMP */ | 452 | #else /* ... !CONFIG_SMP */ |
419 | 453 | ||
420 | static inline void __percpu *mod_percpu(struct module *mod) | 454 | static inline void __percpu *mod_percpu(struct module *mod) |
@@ -441,6 +475,10 @@ static inline void percpu_modcopy(struct module *mod, | |||
441 | /* pcpusec should be 0, and size of that section should be 0. */ | 475 | /* pcpusec should be 0, and size of that section should be 0. */ |
442 | BUG_ON(size != 0); | 476 | BUG_ON(size != 0); |
443 | } | 477 | } |
478 | bool is_module_percpu_address(unsigned long addr) | ||
479 | { | ||
480 | return false; | ||
481 | } | ||
444 | 482 | ||
445 | #endif /* CONFIG_SMP */ | 483 | #endif /* CONFIG_SMP */ |
446 | 484 | ||
diff --git a/mm/percpu.c b/mm/percpu.c index 768419d44ad7..6e09741ddc62 100644 --- a/mm/percpu.c +++ b/mm/percpu.c | |||
@@ -1304,6 +1304,32 @@ void free_percpu(void __percpu *ptr) | |||
1304 | EXPORT_SYMBOL_GPL(free_percpu); | 1304 | EXPORT_SYMBOL_GPL(free_percpu); |
1305 | 1305 | ||
1306 | /** | 1306 | /** |
1307 | * is_kernel_percpu_address - test whether address is from static percpu area | ||
1308 | * @addr: address to test | ||
1309 | * | ||
1310 | * Test whether @addr belongs to in-kernel static percpu area. Module | ||
1311 | * static percpu areas are not considered. For those, use | ||
1312 | * is_module_percpu_address(). | ||
1313 | * | ||
1314 | * RETURNS: | ||
1315 | * %true if @addr is from in-kernel static percpu area, %false otherwise. | ||
1316 | */ | ||
1317 | bool is_kernel_percpu_address(unsigned long addr) | ||
1318 | { | ||
1319 | const size_t static_size = __per_cpu_end - __per_cpu_start; | ||
1320 | void __percpu *base = __addr_to_pcpu_ptr(pcpu_base_addr); | ||
1321 | unsigned int cpu; | ||
1322 | |||
1323 | for_each_possible_cpu(cpu) { | ||
1324 | void *start = per_cpu_ptr(base, cpu); | ||
1325 | |||
1326 | if ((void *)addr >= start && (void *)addr < start + static_size) | ||
1327 | return true; | ||
1328 | } | ||
1329 | return false; | ||
1330 | } | ||
1331 | |||
1332 | /** | ||
1307 | * per_cpu_ptr_to_phys - convert translated percpu address to physical address | 1333 | * per_cpu_ptr_to_phys - convert translated percpu address to physical address |
1308 | * @addr: the address to be converted to physical address | 1334 | * @addr: the address to be converted to physical address |
1309 | * | 1335 | * |