aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--include/linux/module.h1
-rw-r--r--include/linux/percpu.h7
-rw-r--r--kernel/lockdep.c21
-rw-r--r--kernel/module.c38
-rw-r--r--mm/percpu.c26
5 files changed, 77 insertions, 16 deletions
diff --git a/include/linux/module.h b/include/linux/module.h
index 87d247ac6761..f0e2659f4e3e 100644
--- a/include/linux/module.h
+++ b/include/linux/module.h
@@ -395,6 +395,7 @@ static inline int module_is_live(struct module *mod)
395struct module *__module_text_address(unsigned long addr); 395struct module *__module_text_address(unsigned long addr);
396struct module *__module_address(unsigned long addr); 396struct module *__module_address(unsigned long addr);
397bool is_module_address(unsigned long addr); 397bool is_module_address(unsigned long addr);
398bool is_module_percpu_address(unsigned long addr);
398bool is_module_text_address(unsigned long addr); 399bool is_module_text_address(unsigned long addr);
399 400
400static inline int within_module_core(unsigned long addr, struct module *mod) 401static inline int within_module_core(unsigned long addr, struct module *mod)
diff --git a/include/linux/percpu.h b/include/linux/percpu.h
index a93e5bfdccb8..11d5f834b54a 100644
--- a/include/linux/percpu.h
+++ b/include/linux/percpu.h
@@ -137,6 +137,7 @@ extern int __init pcpu_page_first_chunk(size_t reserved_size,
137extern void __percpu *__alloc_reserved_percpu(size_t size, size_t align); 137extern void __percpu *__alloc_reserved_percpu(size_t size, size_t align);
138extern void __percpu *__alloc_percpu(size_t size, size_t align); 138extern void __percpu *__alloc_percpu(size_t size, size_t align);
139extern void free_percpu(void __percpu *__pdata); 139extern void free_percpu(void __percpu *__pdata);
140extern bool is_kernel_percpu_address(unsigned long addr);
140extern phys_addr_t per_cpu_ptr_to_phys(void *addr); 141extern phys_addr_t per_cpu_ptr_to_phys(void *addr);
141 142
142#ifndef CONFIG_HAVE_SETUP_PER_CPU_AREA 143#ifndef CONFIG_HAVE_SETUP_PER_CPU_AREA
@@ -163,6 +164,12 @@ static inline void free_percpu(void __percpu *p)
163 kfree(p); 164 kfree(p);
164} 165}
165 166
167/* can't distinguish from other static vars, always false */
168static inline bool is_kernel_percpu_address(unsigned long addr)
169{
170 return false;
171}
172
166static inline phys_addr_t per_cpu_ptr_to_phys(void *addr) 173static inline phys_addr_t per_cpu_ptr_to_phys(void *addr)
167{ 174{
168 return __pa(addr); 175 return __pa(addr);
diff --git a/kernel/lockdep.c b/kernel/lockdep.c
index c927a549db2c..9bbb9c841e48 100644
--- a/kernel/lockdep.c
+++ b/kernel/lockdep.c
@@ -582,9 +582,6 @@ static int static_obj(void *obj)
582 unsigned long start = (unsigned long) &_stext, 582 unsigned long start = (unsigned long) &_stext,
583 end = (unsigned long) &_end, 583 end = (unsigned long) &_end,
584 addr = (unsigned long) obj; 584 addr = (unsigned long) obj;
585#ifdef CONFIG_SMP
586 int i;
587#endif
588 585
589 /* 586 /*
590 * static variable? 587 * static variable?
@@ -595,24 +592,16 @@ static int static_obj(void *obj)
595 if (arch_is_kernel_data(addr)) 592 if (arch_is_kernel_data(addr))
596 return 1; 593 return 1;
597 594
598#ifdef CONFIG_SMP
599 /* 595 /*
600 * percpu var? 596 * in-kernel percpu var?
601 */ 597 */
602 for_each_possible_cpu(i) { 598 if (is_kernel_percpu_address(addr))
603 start = (unsigned long) &__per_cpu_start + per_cpu_offset(i); 599 return 1;
604 end = (unsigned long) &__per_cpu_start + PERCPU_ENOUGH_ROOM
605 + per_cpu_offset(i);
606
607 if ((addr >= start) && (addr < end))
608 return 1;
609 }
610#endif
611 600
612 /* 601 /*
613 * module var? 602 * module static or percpu var?
614 */ 603 */
615 return is_module_address(addr); 604 return is_module_address(addr) || is_module_percpu_address(addr);
616} 605}
617 606
618/* 607/*
diff --git a/kernel/module.c b/kernel/module.c
index e7a6e53fc73e..9f8d23d8b3a8 100644
--- a/kernel/module.c
+++ b/kernel/module.c
@@ -415,6 +415,40 @@ static void percpu_modcopy(struct module *mod,
415 memcpy(per_cpu_ptr(mod->percpu, cpu), from, size); 415 memcpy(per_cpu_ptr(mod->percpu, cpu), from, size);
416} 416}
417 417
418/**
419 * is_module_percpu_address - test whether address is from module static percpu
420 * @addr: address to test
421 *
422 * Test whether @addr belongs to module static percpu area.
423 *
424 * RETURNS:
425 * %true if @addr is from module static percpu area
426 */
427bool is_module_percpu_address(unsigned long addr)
428{
429 struct module *mod;
430 unsigned int cpu;
431
432 preempt_disable();
433
434 list_for_each_entry_rcu(mod, &modules, list) {
435 if (!mod->percpu_size)
436 continue;
437 for_each_possible_cpu(cpu) {
438 void *start = per_cpu_ptr(mod->percpu, cpu);
439
440 if ((void *)addr >= start &&
441 (void *)addr < start + mod->percpu_size) {
442 preempt_enable();
443 return true;
444 }
445 }
446 }
447
448 preempt_enable();
449 return false;
450}
451
418#else /* ... !CONFIG_SMP */ 452#else /* ... !CONFIG_SMP */
419 453
420static inline void __percpu *mod_percpu(struct module *mod) 454static inline void __percpu *mod_percpu(struct module *mod)
@@ -441,6 +475,10 @@ static inline void percpu_modcopy(struct module *mod,
441 /* pcpusec should be 0, and size of that section should be 0. */ 475 /* pcpusec should be 0, and size of that section should be 0. */
442 BUG_ON(size != 0); 476 BUG_ON(size != 0);
443} 477}
478bool is_module_percpu_address(unsigned long addr)
479{
480 return false;
481}
444 482
445#endif /* CONFIG_SMP */ 483#endif /* CONFIG_SMP */
446 484
diff --git a/mm/percpu.c b/mm/percpu.c
index 768419d44ad7..6e09741ddc62 100644
--- a/mm/percpu.c
+++ b/mm/percpu.c
@@ -1304,6 +1304,32 @@ void free_percpu(void __percpu *ptr)
1304EXPORT_SYMBOL_GPL(free_percpu); 1304EXPORT_SYMBOL_GPL(free_percpu);
1305 1305
1306/** 1306/**
1307 * is_kernel_percpu_address - test whether address is from static percpu area
1308 * @addr: address to test
1309 *
1310 * Test whether @addr belongs to in-kernel static percpu area. Module
1311 * static percpu areas are not considered. For those, use
1312 * is_module_percpu_address().
1313 *
1314 * RETURNS:
1315 * %true if @addr is from in-kernel static percpu area, %false otherwise.
1316 */
1317bool is_kernel_percpu_address(unsigned long addr)
1318{
1319 const size_t static_size = __per_cpu_end - __per_cpu_start;
1320 void __percpu *base = __addr_to_pcpu_ptr(pcpu_base_addr);
1321 unsigned int cpu;
1322
1323 for_each_possible_cpu(cpu) {
1324 void *start = per_cpu_ptr(base, cpu);
1325
1326 if ((void *)addr >= start && (void *)addr < start + static_size)
1327 return true;
1328 }
1329 return false;
1330}
1331
1332/**
1307 * per_cpu_ptr_to_phys - convert translated percpu address to physical address 1333 * per_cpu_ptr_to_phys - convert translated percpu address to physical address
1308 * @addr: the address to be converted to physical address 1334 * @addr: the address to be converted to physical address
1309 * 1335 *