diff options
-rw-r--r-- | include/linux/module.h | 16 | ||||
-rw-r--r-- | include/trace/events/module.h | 2 | ||||
-rw-r--r-- | kernel/module.c | 39 |
3 files changed, 7 insertions, 50 deletions
diff --git a/include/linux/module.h b/include/linux/module.h index 71f282a4e307..ebfb0e153c6a 100644 --- a/include/linux/module.h +++ b/include/linux/module.h | |||
@@ -210,20 +210,6 @@ enum module_state { | |||
210 | MODULE_STATE_UNFORMED, /* Still setting it up. */ | 210 | MODULE_STATE_UNFORMED, /* Still setting it up. */ |
211 | }; | 211 | }; |
212 | 212 | ||
213 | /** | ||
214 | * struct module_ref - per cpu module reference counts | ||
215 | * @incs: number of module get on this cpu | ||
216 | * @decs: number of module put on this cpu | ||
217 | * | ||
218 | * We force an alignment on 8 or 16 bytes, so that alloc_percpu() | ||
219 | * put @incs/@decs in same cache line, with no extra memory cost, | ||
220 | * since alloc_percpu() is fine grained. | ||
221 | */ | ||
222 | struct module_ref { | ||
223 | unsigned long incs; | ||
224 | unsigned long decs; | ||
225 | } __attribute((aligned(2 * sizeof(unsigned long)))); | ||
226 | |||
227 | struct module { | 213 | struct module { |
228 | enum module_state state; | 214 | enum module_state state; |
229 | 215 | ||
@@ -367,7 +353,7 @@ struct module { | |||
367 | /* Destruction function. */ | 353 | /* Destruction function. */ |
368 | void (*exit)(void); | 354 | void (*exit)(void); |
369 | 355 | ||
370 | struct module_ref __percpu *refptr; | 356 | atomic_t refcnt; |
371 | #endif | 357 | #endif |
372 | 358 | ||
373 | #ifdef CONFIG_CONSTRUCTORS | 359 | #ifdef CONFIG_CONSTRUCTORS |
diff --git a/include/trace/events/module.h b/include/trace/events/module.h index 7c5cbfe3fc49..81c4c183d348 100644 --- a/include/trace/events/module.h +++ b/include/trace/events/module.h | |||
@@ -80,7 +80,7 @@ DECLARE_EVENT_CLASS(module_refcnt, | |||
80 | 80 | ||
81 | TP_fast_assign( | 81 | TP_fast_assign( |
82 | __entry->ip = ip; | 82 | __entry->ip = ip; |
83 | __entry->refcnt = __this_cpu_read(mod->refptr->incs) - __this_cpu_read(mod->refptr->decs); | 83 | __entry->refcnt = atomic_read(&mod->refcnt); |
84 | __assign_str(name, mod->name); | 84 | __assign_str(name, mod->name); |
85 | ), | 85 | ), |
86 | 86 | ||
diff --git a/kernel/module.c b/kernel/module.c index d596a306b0a1..b1d485df5ac1 100644 --- a/kernel/module.c +++ b/kernel/module.c | |||
@@ -631,15 +631,11 @@ EXPORT_TRACEPOINT_SYMBOL(module_get); | |||
631 | /* Init the unload section of the module. */ | 631 | /* Init the unload section of the module. */ |
632 | static int module_unload_init(struct module *mod) | 632 | static int module_unload_init(struct module *mod) |
633 | { | 633 | { |
634 | mod->refptr = alloc_percpu(struct module_ref); | ||
635 | if (!mod->refptr) | ||
636 | return -ENOMEM; | ||
637 | |||
638 | INIT_LIST_HEAD(&mod->source_list); | 634 | INIT_LIST_HEAD(&mod->source_list); |
639 | INIT_LIST_HEAD(&mod->target_list); | 635 | INIT_LIST_HEAD(&mod->target_list); |
640 | 636 | ||
641 | /* Hold reference count during initialization. */ | 637 | /* Hold reference count during initialization. */ |
642 | raw_cpu_write(mod->refptr->incs, 1); | 638 | atomic_set(&mod->refcnt, 1); |
643 | 639 | ||
644 | return 0; | 640 | return 0; |
645 | } | 641 | } |
@@ -721,8 +717,6 @@ static void module_unload_free(struct module *mod) | |||
721 | kfree(use); | 717 | kfree(use); |
722 | } | 718 | } |
723 | mutex_unlock(&module_mutex); | 719 | mutex_unlock(&module_mutex); |
724 | |||
725 | free_percpu(mod->refptr); | ||
726 | } | 720 | } |
727 | 721 | ||
728 | #ifdef CONFIG_MODULE_FORCE_UNLOAD | 722 | #ifdef CONFIG_MODULE_FORCE_UNLOAD |
@@ -772,28 +766,7 @@ static int try_stop_module(struct module *mod, int flags, int *forced) | |||
772 | 766 | ||
773 | unsigned long module_refcount(struct module *mod) | 767 | unsigned long module_refcount(struct module *mod) |
774 | { | 768 | { |
775 | unsigned long incs = 0, decs = 0; | 769 | return (unsigned long)atomic_read(&mod->refcnt); |
776 | int cpu; | ||
777 | |||
778 | for_each_possible_cpu(cpu) | ||
779 | decs += per_cpu_ptr(mod->refptr, cpu)->decs; | ||
780 | /* | ||
781 | * ensure the incs are added up after the decs. | ||
782 | * module_put ensures incs are visible before decs with smp_wmb. | ||
783 | * | ||
784 | * This 2-count scheme avoids the situation where the refcount | ||
785 | * for CPU0 is read, then CPU0 increments the module refcount, | ||
786 | * then CPU1 drops that refcount, then the refcount for CPU1 is | ||
787 | * read. We would record a decrement but not its corresponding | ||
788 | * increment so we would see a low count (disaster). | ||
789 | * | ||
790 | * Rare situation? But module_refcount can be preempted, and we | ||
791 | * might be tallying up 4096+ CPUs. So it is not impossible. | ||
792 | */ | ||
793 | smp_rmb(); | ||
794 | for_each_possible_cpu(cpu) | ||
795 | incs += per_cpu_ptr(mod->refptr, cpu)->incs; | ||
796 | return incs - decs; | ||
797 | } | 770 | } |
798 | EXPORT_SYMBOL(module_refcount); | 771 | EXPORT_SYMBOL(module_refcount); |
799 | 772 | ||
@@ -935,7 +908,7 @@ void __module_get(struct module *module) | |||
935 | { | 908 | { |
936 | if (module) { | 909 | if (module) { |
937 | preempt_disable(); | 910 | preempt_disable(); |
938 | __this_cpu_inc(module->refptr->incs); | 911 | atomic_inc(&module->refcnt); |
939 | trace_module_get(module, _RET_IP_); | 912 | trace_module_get(module, _RET_IP_); |
940 | preempt_enable(); | 913 | preempt_enable(); |
941 | } | 914 | } |
@@ -950,7 +923,7 @@ bool try_module_get(struct module *module) | |||
950 | preempt_disable(); | 923 | preempt_disable(); |
951 | 924 | ||
952 | if (likely(module_is_live(module))) { | 925 | if (likely(module_is_live(module))) { |
953 | __this_cpu_inc(module->refptr->incs); | 926 | atomic_inc(&module->refcnt); |
954 | trace_module_get(module, _RET_IP_); | 927 | trace_module_get(module, _RET_IP_); |
955 | } else | 928 | } else |
956 | ret = false; | 929 | ret = false; |
@@ -965,9 +938,7 @@ void module_put(struct module *module) | |||
965 | { | 938 | { |
966 | if (module) { | 939 | if (module) { |
967 | preempt_disable(); | 940 | preempt_disable(); |
968 | smp_wmb(); /* see comment in module_refcount */ | 941 | atomic_dec(&module->refcnt); |
969 | __this_cpu_inc(module->refptr->decs); | ||
970 | |||
971 | trace_module_put(module, _RET_IP_); | 942 | trace_module_put(module, _RET_IP_); |
972 | preempt_enable(); | 943 | preempt_enable(); |
973 | } | 944 | } |