aboutsummaryrefslogtreecommitdiffstats
path: root/kernel/module.c
diff options
context:
space:
mode:
Diffstat (limited to 'kernel/module.c')
-rw-r--r--kernel/module.c1382
1 files changed, 778 insertions, 604 deletions
diff --git a/kernel/module.c b/kernel/module.c
index 1016b75b026a..d0b5f8db11b4 100644
--- a/kernel/module.c
+++ b/kernel/module.c
@@ -1,6 +1,6 @@
1/* 1/*
2 Copyright (C) 2002 Richard Henderson 2 Copyright (C) 2002 Richard Henderson
3 Copyright (C) 2001 Rusty Russell, 2002 Rusty Russell IBM. 3 Copyright (C) 2001 Rusty Russell, 2002, 2010 Rusty Russell IBM.
4 4
5 This program is free software; you can redistribute it and/or modify 5 This program is free software; you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published by 6 it under the terms of the GNU General Public License as published by
@@ -59,8 +59,6 @@
59#define CREATE_TRACE_POINTS 59#define CREATE_TRACE_POINTS
60#include <trace/events/module.h> 60#include <trace/events/module.h>
61 61
62EXPORT_TRACEPOINT_SYMBOL(module_get);
63
64#if 0 62#if 0
65#define DEBUGP printk 63#define DEBUGP printk
66#else 64#else
@@ -74,11 +72,19 @@ EXPORT_TRACEPOINT_SYMBOL(module_get);
74/* If this is set, the section belongs in the init part of the module */ 72/* If this is set, the section belongs in the init part of the module */
75#define INIT_OFFSET_MASK (1UL << (BITS_PER_LONG-1)) 73#define INIT_OFFSET_MASK (1UL << (BITS_PER_LONG-1))
76 74
77/* List of modules, protected by module_mutex or preempt_disable 75/*
76 * Mutex protects:
77 * 1) List of modules (also safely readable with preempt_disable),
78 * 2) module_use links,
79 * 3) module_addr_min/module_addr_max.
78 * (delete uses stop_machine/add uses RCU list operations). */ 80 * (delete uses stop_machine/add uses RCU list operations). */
79DEFINE_MUTEX(module_mutex); 81DEFINE_MUTEX(module_mutex);
80EXPORT_SYMBOL_GPL(module_mutex); 82EXPORT_SYMBOL_GPL(module_mutex);
81static LIST_HEAD(modules); 83static LIST_HEAD(modules);
84#ifdef CONFIG_KGDB_KDB
85struct list_head *kdb_modules = &modules; /* kdb needs the list of modules */
86#endif /* CONFIG_KGDB_KDB */
87
82 88
83/* Block module loading/unloading? */ 89/* Block module loading/unloading? */
84int modules_disabled = 0; 90int modules_disabled = 0;
@@ -88,7 +94,8 @@ static DECLARE_WAIT_QUEUE_HEAD(module_wq);
88 94
89static BLOCKING_NOTIFIER_HEAD(module_notify_list); 95static BLOCKING_NOTIFIER_HEAD(module_notify_list);
90 96
91/* Bounds of module allocation, for speeding __module_address */ 97/* Bounds of module allocation, for speeding __module_address.
98 * Protected by module_mutex. */
92static unsigned long module_addr_min = -1UL, module_addr_max = 0; 99static unsigned long module_addr_min = -1UL, module_addr_max = 0;
93 100
94int register_module_notifier(struct notifier_block * nb) 101int register_module_notifier(struct notifier_block * nb)
@@ -103,6 +110,20 @@ int unregister_module_notifier(struct notifier_block * nb)
103} 110}
104EXPORT_SYMBOL(unregister_module_notifier); 111EXPORT_SYMBOL(unregister_module_notifier);
105 112
113struct load_info {
114 Elf_Ehdr *hdr;
115 unsigned long len;
116 Elf_Shdr *sechdrs;
117 char *secstrings, *strtab;
118 unsigned long *strmap;
119 unsigned long symoffs, stroffs;
120 struct _ddebug *debug;
121 unsigned int num_debug;
122 struct {
123 unsigned int sym, str, mod, vers, info, pcpu;
124 } index;
125};
126
106/* We require a truly strong try_module_get(): 0 means failure due to 127/* We require a truly strong try_module_get(): 0 means failure due to
107 ongoing or failed initialization etc. */ 128 ongoing or failed initialization etc. */
108static inline int strong_try_module_get(struct module *mod) 129static inline int strong_try_module_get(struct module *mod)
@@ -133,42 +154,38 @@ void __module_put_and_exit(struct module *mod, long code)
133EXPORT_SYMBOL(__module_put_and_exit); 154EXPORT_SYMBOL(__module_put_and_exit);
134 155
135/* Find a module section: 0 means not found. */ 156/* Find a module section: 0 means not found. */
136static unsigned int find_sec(Elf_Ehdr *hdr, 157static unsigned int find_sec(const struct load_info *info, const char *name)
137 Elf_Shdr *sechdrs,
138 const char *secstrings,
139 const char *name)
140{ 158{
141 unsigned int i; 159 unsigned int i;
142 160
143 for (i = 1; i < hdr->e_shnum; i++) 161 for (i = 1; i < info->hdr->e_shnum; i++) {
162 Elf_Shdr *shdr = &info->sechdrs[i];
144 /* Alloc bit cleared means "ignore it." */ 163 /* Alloc bit cleared means "ignore it." */
145 if ((sechdrs[i].sh_flags & SHF_ALLOC) 164 if ((shdr->sh_flags & SHF_ALLOC)
146 && strcmp(secstrings+sechdrs[i].sh_name, name) == 0) 165 && strcmp(info->secstrings + shdr->sh_name, name) == 0)
147 return i; 166 return i;
167 }
148 return 0; 168 return 0;
149} 169}
150 170
151/* Find a module section, or NULL. */ 171/* Find a module section, or NULL. */
152static void *section_addr(Elf_Ehdr *hdr, Elf_Shdr *shdrs, 172static void *section_addr(const struct load_info *info, const char *name)
153 const char *secstrings, const char *name)
154{ 173{
155 /* Section 0 has sh_addr 0. */ 174 /* Section 0 has sh_addr 0. */
156 return (void *)shdrs[find_sec(hdr, shdrs, secstrings, name)].sh_addr; 175 return (void *)info->sechdrs[find_sec(info, name)].sh_addr;
157} 176}
158 177
159/* Find a module section, or NULL. Fill in number of "objects" in section. */ 178/* Find a module section, or NULL. Fill in number of "objects" in section. */
160static void *section_objs(Elf_Ehdr *hdr, 179static void *section_objs(const struct load_info *info,
161 Elf_Shdr *sechdrs,
162 const char *secstrings,
163 const char *name, 180 const char *name,
164 size_t object_size, 181 size_t object_size,
165 unsigned int *num) 182 unsigned int *num)
166{ 183{
167 unsigned int sec = find_sec(hdr, sechdrs, secstrings, name); 184 unsigned int sec = find_sec(info, name);
168 185
169 /* Section 0 has sh_addr 0 and sh_size 0. */ 186 /* Section 0 has sh_addr 0 and sh_size 0. */
170 *num = sechdrs[sec].sh_size / object_size; 187 *num = info->sechdrs[sec].sh_size / object_size;
171 return (void *)sechdrs[sec].sh_addr; 188 return (void *)info->sechdrs[sec].sh_addr;
172} 189}
173 190
174/* Provided by the linker */ 191/* Provided by the linker */
@@ -178,8 +195,6 @@ extern const struct kernel_symbol __start___ksymtab_gpl[];
178extern const struct kernel_symbol __stop___ksymtab_gpl[]; 195extern const struct kernel_symbol __stop___ksymtab_gpl[];
179extern const struct kernel_symbol __start___ksymtab_gpl_future[]; 196extern const struct kernel_symbol __start___ksymtab_gpl_future[];
180extern const struct kernel_symbol __stop___ksymtab_gpl_future[]; 197extern const struct kernel_symbol __stop___ksymtab_gpl_future[];
181extern const struct kernel_symbol __start___ksymtab_gpl_future[];
182extern const struct kernel_symbol __stop___ksymtab_gpl_future[];
183extern const unsigned long __start___kcrctab[]; 198extern const unsigned long __start___kcrctab[];
184extern const unsigned long __start___kcrctab_gpl[]; 199extern const unsigned long __start___kcrctab_gpl[];
185extern const unsigned long __start___kcrctab_gpl_future[]; 200extern const unsigned long __start___kcrctab_gpl_future[];
@@ -222,7 +237,7 @@ bool each_symbol(bool (*fn)(const struct symsearch *arr, struct module *owner,
222 unsigned int symnum, void *data), void *data) 237 unsigned int symnum, void *data), void *data)
223{ 238{
224 struct module *mod; 239 struct module *mod;
225 const struct symsearch arr[] = { 240 static const struct symsearch arr[] = {
226 { __start___ksymtab, __stop___ksymtab, __start___kcrctab, 241 { __start___ksymtab, __stop___ksymtab, __start___kcrctab,
227 NOT_GPL_ONLY, false }, 242 NOT_GPL_ONLY, false },
228 { __start___ksymtab_gpl, __stop___ksymtab_gpl, 243 { __start___ksymtab_gpl, __stop___ksymtab_gpl,
@@ -329,7 +344,7 @@ static bool find_symbol_in_section(const struct symsearch *syms,
329} 344}
330 345
331/* Find a symbol and return it, along with, (optional) crc and 346/* Find a symbol and return it, along with, (optional) crc and
332 * (optional) module which owns it */ 347 * (optional) module which owns it. Needs preempt disabled or module_mutex. */
333const struct kernel_symbol *find_symbol(const char *name, 348const struct kernel_symbol *find_symbol(const char *name,
334 struct module **owner, 349 struct module **owner,
335 const unsigned long **crc, 350 const unsigned long **crc,
@@ -387,7 +402,8 @@ static int percpu_modalloc(struct module *mod,
387 mod->percpu = __alloc_reserved_percpu(size, align); 402 mod->percpu = __alloc_reserved_percpu(size, align);
388 if (!mod->percpu) { 403 if (!mod->percpu) {
389 printk(KERN_WARNING 404 printk(KERN_WARNING
390 "Could not allocate %lu bytes percpu data\n", size); 405 "%s: Could not allocate %lu bytes percpu data\n",
406 mod->name, size);
391 return -ENOMEM; 407 return -ENOMEM;
392 } 408 }
393 mod->percpu_size = size; 409 mod->percpu_size = size;
@@ -399,11 +415,9 @@ static void percpu_modfree(struct module *mod)
399 free_percpu(mod->percpu); 415 free_percpu(mod->percpu);
400} 416}
401 417
402static unsigned int find_pcpusec(Elf_Ehdr *hdr, 418static unsigned int find_pcpusec(struct load_info *info)
403 Elf_Shdr *sechdrs,
404 const char *secstrings)
405{ 419{
406 return find_sec(hdr, sechdrs, secstrings, ".data.percpu"); 420 return find_sec(info, ".data..percpu");
407} 421}
408 422
409static void percpu_modcopy(struct module *mod, 423static void percpu_modcopy(struct module *mod,
@@ -463,9 +477,7 @@ static inline int percpu_modalloc(struct module *mod,
463static inline void percpu_modfree(struct module *mod) 477static inline void percpu_modfree(struct module *mod)
464{ 478{
465} 479}
466static inline unsigned int find_pcpusec(Elf_Ehdr *hdr, 480static unsigned int find_pcpusec(struct load_info *info)
467 Elf_Shdr *sechdrs,
468 const char *secstrings)
469{ 481{
470 return 0; 482 return 0;
471} 483}
@@ -515,37 +527,34 @@ MODINFO_ATTR(srcversion);
515static char last_unloaded_module[MODULE_NAME_LEN+1]; 527static char last_unloaded_module[MODULE_NAME_LEN+1];
516 528
517#ifdef CONFIG_MODULE_UNLOAD 529#ifdef CONFIG_MODULE_UNLOAD
530
531EXPORT_TRACEPOINT_SYMBOL(module_get);
532
518/* Init the unload section of the module. */ 533/* Init the unload section of the module. */
519static void module_unload_init(struct module *mod) 534static int module_unload_init(struct module *mod)
520{ 535{
521 int cpu; 536 mod->refptr = alloc_percpu(struct module_ref);
537 if (!mod->refptr)
538 return -ENOMEM;
522 539
523 INIT_LIST_HEAD(&mod->modules_which_use_me); 540 INIT_LIST_HEAD(&mod->source_list);
524 for_each_possible_cpu(cpu) { 541 INIT_LIST_HEAD(&mod->target_list);
525 per_cpu_ptr(mod->refptr, cpu)->incs = 0;
526 per_cpu_ptr(mod->refptr, cpu)->decs = 0;
527 }
528 542
529 /* Hold reference count during initialization. */ 543 /* Hold reference count during initialization. */
530 __this_cpu_write(mod->refptr->incs, 1); 544 __this_cpu_write(mod->refptr->incs, 1);
531 /* Backwards compatibility macros put refcount during init. */ 545 /* Backwards compatibility macros put refcount during init. */
532 mod->waiter = current; 546 mod->waiter = current;
533}
534 547
535/* modules using other modules */ 548 return 0;
536struct module_use 549}
537{
538 struct list_head list;
539 struct module *module_which_uses;
540};
541 550
542/* Does a already use b? */ 551/* Does a already use b? */
543static int already_uses(struct module *a, struct module *b) 552static int already_uses(struct module *a, struct module *b)
544{ 553{
545 struct module_use *use; 554 struct module_use *use;
546 555
547 list_for_each_entry(use, &b->modules_which_use_me, list) { 556 list_for_each_entry(use, &b->source_list, source_list) {
548 if (use->module_which_uses == a) { 557 if (use->source == a) {
549 DEBUGP("%s uses %s!\n", a->name, b->name); 558 DEBUGP("%s uses %s!\n", a->name, b->name);
550 return 1; 559 return 1;
551 } 560 }
@@ -554,62 +563,70 @@ static int already_uses(struct module *a, struct module *b)
554 return 0; 563 return 0;
555} 564}
556 565
557/* Module a uses b */ 566/*
558int use_module(struct module *a, struct module *b) 567 * Module a uses b
568 * - we add 'a' as a "source", 'b' as a "target" of module use
569 * - the module_use is added to the list of 'b' sources (so
570 * 'b' can walk the list to see who sourced them), and of 'a'
571 * targets (so 'a' can see what modules it targets).
572 */
573static int add_module_usage(struct module *a, struct module *b)
559{ 574{
560 struct module_use *use; 575 struct module_use *use;
561 int no_warn, err;
562 576
563 if (b == NULL || already_uses(a, b)) return 1; 577 DEBUGP("Allocating new usage for %s.\n", a->name);
578 use = kmalloc(sizeof(*use), GFP_ATOMIC);
579 if (!use) {
580 printk(KERN_WARNING "%s: out of memory loading\n", a->name);
581 return -ENOMEM;
582 }
583
584 use->source = a;
585 use->target = b;
586 list_add(&use->source_list, &b->source_list);
587 list_add(&use->target_list, &a->target_list);
588 return 0;
589}
590
591/* Module a uses b: caller needs module_mutex() */
592int ref_module(struct module *a, struct module *b)
593{
594 int err;
564 595
565 /* If we're interrupted or time out, we fail. */ 596 if (b == NULL || already_uses(a, b))
566 if (wait_event_interruptible_timeout(
567 module_wq, (err = strong_try_module_get(b)) != -EBUSY,
568 30 * HZ) <= 0) {
569 printk("%s: gave up waiting for init of module %s.\n",
570 a->name, b->name);
571 return 0; 597 return 0;
572 }
573 598
574 /* If strong_try_module_get() returned a different error, we fail. */ 599 /* If module isn't available, we fail. */
600 err = strong_try_module_get(b);
575 if (err) 601 if (err)
576 return 0; 602 return err;
577 603
578 DEBUGP("Allocating new usage for %s.\n", a->name); 604 err = add_module_usage(a, b);
579 use = kmalloc(sizeof(*use), GFP_ATOMIC); 605 if (err) {
580 if (!use) {
581 printk("%s: out of memory loading\n", a->name);
582 module_put(b); 606 module_put(b);
583 return 0; 607 return err;
584 } 608 }
585 609 return 0;
586 use->module_which_uses = a;
587 list_add(&use->list, &b->modules_which_use_me);
588 no_warn = sysfs_create_link(b->holders_dir, &a->mkobj.kobj, a->name);
589 return 1;
590} 610}
591EXPORT_SYMBOL_GPL(use_module); 611EXPORT_SYMBOL_GPL(ref_module);
592 612
593/* Clear the unload stuff of the module. */ 613/* Clear the unload stuff of the module. */
594static void module_unload_free(struct module *mod) 614static void module_unload_free(struct module *mod)
595{ 615{
596 struct module *i; 616 struct module_use *use, *tmp;
597 617
598 list_for_each_entry(i, &modules, list) { 618 mutex_lock(&module_mutex);
599 struct module_use *use; 619 list_for_each_entry_safe(use, tmp, &mod->target_list, target_list) {
600 620 struct module *i = use->target;
601 list_for_each_entry(use, &i->modules_which_use_me, list) { 621 DEBUGP("%s unusing %s\n", mod->name, i->name);
602 if (use->module_which_uses == mod) { 622 module_put(i);
603 DEBUGP("%s unusing %s\n", mod->name, i->name); 623 list_del(&use->source_list);
604 module_put(i); 624 list_del(&use->target_list);
605 list_del(&use->list); 625 kfree(use);
606 kfree(use);
607 sysfs_remove_link(i->holders_dir, mod->name);
608 /* There can be at most one match. */
609 break;
610 }
611 }
612 } 626 }
627 mutex_unlock(&module_mutex);
628
629 free_percpu(mod->refptr);
613} 630}
614 631
615#ifdef CONFIG_MODULE_FORCE_UNLOAD 632#ifdef CONFIG_MODULE_FORCE_UNLOAD
@@ -723,16 +740,8 @@ SYSCALL_DEFINE2(delete_module, const char __user *, name_user,
723 return -EFAULT; 740 return -EFAULT;
724 name[MODULE_NAME_LEN-1] = '\0'; 741 name[MODULE_NAME_LEN-1] = '\0';
725 742
726 /* Create stop_machine threads since free_module relies on 743 if (mutex_lock_interruptible(&module_mutex) != 0)
727 * a non-failing stop_machine call. */ 744 return -EINTR;
728 ret = stop_machine_create();
729 if (ret)
730 return ret;
731
732 if (mutex_lock_interruptible(&module_mutex) != 0) {
733 ret = -EINTR;
734 goto out_stop;
735 }
736 745
737 mod = find_module(name); 746 mod = find_module(name);
738 if (!mod) { 747 if (!mod) {
@@ -740,7 +749,7 @@ SYSCALL_DEFINE2(delete_module, const char __user *, name_user,
740 goto out; 749 goto out;
741 } 750 }
742 751
743 if (!list_empty(&mod->modules_which_use_me)) { 752 if (!list_empty(&mod->source_list)) {
744 /* Other modules depend on us: get rid of them first. */ 753 /* Other modules depend on us: get rid of them first. */
745 ret = -EWOULDBLOCK; 754 ret = -EWOULDBLOCK;
746 goto out; 755 goto out;
@@ -784,16 +793,14 @@ SYSCALL_DEFINE2(delete_module, const char __user *, name_user,
784 blocking_notifier_call_chain(&module_notify_list, 793 blocking_notifier_call_chain(&module_notify_list,
785 MODULE_STATE_GOING, mod); 794 MODULE_STATE_GOING, mod);
786 async_synchronize_full(); 795 async_synchronize_full();
787 mutex_lock(&module_mutex); 796
788 /* Store the name of the last unloaded module for diagnostic purposes */ 797 /* Store the name of the last unloaded module for diagnostic purposes */
789 strlcpy(last_unloaded_module, mod->name, sizeof(last_unloaded_module)); 798 strlcpy(last_unloaded_module, mod->name, sizeof(last_unloaded_module));
790 ddebug_remove_module(mod->name);
791 free_module(mod);
792 799
793 out: 800 free_module(mod);
801 return 0;
802out:
794 mutex_unlock(&module_mutex); 803 mutex_unlock(&module_mutex);
795out_stop:
796 stop_machine_destroy();
797 return ret; 804 return ret;
798} 805}
799 806
@@ -806,9 +813,9 @@ static inline void print_unload_info(struct seq_file *m, struct module *mod)
806 813
807 /* Always include a trailing , so userspace can differentiate 814 /* Always include a trailing , so userspace can differentiate
808 between this and the old multi-field proc format. */ 815 between this and the old multi-field proc format. */
809 list_for_each_entry(use, &mod->modules_which_use_me, list) { 816 list_for_each_entry(use, &mod->source_list, source_list) {
810 printed_something = 1; 817 printed_something = 1;
811 seq_printf(m, "%s,", use->module_which_uses->name); 818 seq_printf(m, "%s,", use->source->name);
812 } 819 }
813 820
814 if (mod->init != NULL && mod->exit == NULL) { 821 if (mod->init != NULL && mod->exit == NULL) {
@@ -867,8 +874,7 @@ void module_put(struct module *module)
867 smp_wmb(); /* see comment in module_refcount */ 874 smp_wmb(); /* see comment in module_refcount */
868 __this_cpu_inc(module->refptr->decs); 875 __this_cpu_inc(module->refptr->decs);
869 876
870 trace_module_put(module, _RET_IP_, 877 trace_module_put(module, _RET_IP_);
871 __this_cpu_read(module->refptr->decs));
872 /* Maybe they're waiting for us to drop reference? */ 878 /* Maybe they're waiting for us to drop reference? */
873 if (unlikely(!module_is_live(module))) 879 if (unlikely(!module_is_live(module)))
874 wake_up_process(module->waiter); 880 wake_up_process(module->waiter);
@@ -888,14 +894,15 @@ static inline void module_unload_free(struct module *mod)
888{ 894{
889} 895}
890 896
891int use_module(struct module *a, struct module *b) 897int ref_module(struct module *a, struct module *b)
892{ 898{
893 return strong_try_module_get(b) == 0; 899 return strong_try_module_get(b);
894} 900}
895EXPORT_SYMBOL_GPL(use_module); 901EXPORT_SYMBOL_GPL(ref_module);
896 902
897static inline void module_unload_init(struct module *mod) 903static inline int module_unload_init(struct module *mod)
898{ 904{
905 return 0;
899} 906}
900#endif /* CONFIG_MODULE_UNLOAD */ 907#endif /* CONFIG_MODULE_UNLOAD */
901 908
@@ -1009,6 +1016,8 @@ static inline int check_modstruct_version(Elf_Shdr *sechdrs,
1009{ 1016{
1010 const unsigned long *crc; 1017 const unsigned long *crc;
1011 1018
1019 /* Since this should be found in kernel (which can't be removed),
1020 * no locking is necessary. */
1012 if (!find_symbol(MODULE_SYMBOL_PREFIX "module_layout", NULL, 1021 if (!find_symbol(MODULE_SYMBOL_PREFIX "module_layout", NULL,
1013 &crc, true, false)) 1022 &crc, true, false))
1014 BUG(); 1023 BUG();
@@ -1051,35 +1060,68 @@ static inline int same_magic(const char *amagic, const char *bmagic,
1051} 1060}
1052#endif /* CONFIG_MODVERSIONS */ 1061#endif /* CONFIG_MODVERSIONS */
1053 1062
1054/* Resolve a symbol for this module. I.e. if we find one, record usage. 1063/* Resolve a symbol for this module. I.e. if we find one, record usage. */
1055 Must be holding module_mutex. */ 1064static const struct kernel_symbol *resolve_symbol(struct module *mod,
1056static const struct kernel_symbol *resolve_symbol(Elf_Shdr *sechdrs, 1065 const struct load_info *info,
1057 unsigned int versindex,
1058 const char *name, 1066 const char *name,
1059 struct module *mod) 1067 char ownername[])
1060{ 1068{
1061 struct module *owner; 1069 struct module *owner;
1062 const struct kernel_symbol *sym; 1070 const struct kernel_symbol *sym;
1063 const unsigned long *crc; 1071 const unsigned long *crc;
1072 int err;
1064 1073
1074 mutex_lock(&module_mutex);
1065 sym = find_symbol(name, &owner, &crc, 1075 sym = find_symbol(name, &owner, &crc,
1066 !(mod->taints & (1 << TAINT_PROPRIETARY_MODULE)), true); 1076 !(mod->taints & (1 << TAINT_PROPRIETARY_MODULE)), true);
1067 /* use_module can fail due to OOM, 1077 if (!sym)
1068 or module initialization or unloading */ 1078 goto unlock;
1069 if (sym) { 1079
1070 if (!check_version(sechdrs, versindex, name, mod, crc, owner) 1080 if (!check_version(info->sechdrs, info->index.vers, name, mod, crc,
1071 || !use_module(mod, owner)) 1081 owner)) {
1072 sym = NULL; 1082 sym = ERR_PTR(-EINVAL);
1083 goto getname;
1084 }
1085
1086 err = ref_module(mod, owner);
1087 if (err) {
1088 sym = ERR_PTR(err);
1089 goto getname;
1073 } 1090 }
1091
1092getname:
1093 /* We must make copy under the lock if we failed to get ref. */
1094 strncpy(ownername, module_name(owner), MODULE_NAME_LEN);
1095unlock:
1096 mutex_unlock(&module_mutex);
1074 return sym; 1097 return sym;
1075} 1098}
1076 1099
1100static const struct kernel_symbol *
1101resolve_symbol_wait(struct module *mod,
1102 const struct load_info *info,
1103 const char *name)
1104{
1105 const struct kernel_symbol *ksym;
1106 char owner[MODULE_NAME_LEN];
1107
1108 if (wait_event_interruptible_timeout(module_wq,
1109 !IS_ERR(ksym = resolve_symbol(mod, info, name, owner))
1110 || PTR_ERR(ksym) != -EBUSY,
1111 30 * HZ) <= 0) {
1112 printk(KERN_WARNING "%s: gave up waiting for init of module %s.\n",
1113 mod->name, owner);
1114 }
1115 return ksym;
1116}
1117
1077/* 1118/*
1078 * /sys/module/foo/sections stuff 1119 * /sys/module/foo/sections stuff
1079 * J. Corbet <corbet@lwn.net> 1120 * J. Corbet <corbet@lwn.net>
1080 */ 1121 */
1081#if defined(CONFIG_KALLSYMS) && defined(CONFIG_SYSFS) 1122#ifdef CONFIG_SYSFS
1082 1123
1124#ifdef CONFIG_KALLSYMS
1083static inline bool sect_empty(const Elf_Shdr *sect) 1125static inline bool sect_empty(const Elf_Shdr *sect)
1084{ 1126{
1085 return !(sect->sh_flags & SHF_ALLOC) || sect->sh_size == 0; 1127 return !(sect->sh_flags & SHF_ALLOC) || sect->sh_size == 0;
@@ -1116,8 +1158,7 @@ static void free_sect_attrs(struct module_sect_attrs *sect_attrs)
1116 kfree(sect_attrs); 1158 kfree(sect_attrs);
1117} 1159}
1118 1160
1119static void add_sect_attrs(struct module *mod, unsigned int nsect, 1161static void add_sect_attrs(struct module *mod, const struct load_info *info)
1120 char *secstrings, Elf_Shdr *sechdrs)
1121{ 1162{
1122 unsigned int nloaded = 0, i, size[2]; 1163 unsigned int nloaded = 0, i, size[2];
1123 struct module_sect_attrs *sect_attrs; 1164 struct module_sect_attrs *sect_attrs;
@@ -1125,8 +1166,8 @@ static void add_sect_attrs(struct module *mod, unsigned int nsect,
1125 struct attribute **gattr; 1166 struct attribute **gattr;
1126 1167
1127 /* Count loaded sections and allocate structures */ 1168 /* Count loaded sections and allocate structures */
1128 for (i = 0; i < nsect; i++) 1169 for (i = 0; i < info->hdr->e_shnum; i++)
1129 if (!sect_empty(&sechdrs[i])) 1170 if (!sect_empty(&info->sechdrs[i]))
1130 nloaded++; 1171 nloaded++;
1131 size[0] = ALIGN(sizeof(*sect_attrs) 1172 size[0] = ALIGN(sizeof(*sect_attrs)
1132 + nloaded * sizeof(sect_attrs->attrs[0]), 1173 + nloaded * sizeof(sect_attrs->attrs[0]),
@@ -1143,11 +1184,12 @@ static void add_sect_attrs(struct module *mod, unsigned int nsect,
1143 sect_attrs->nsections = 0; 1184 sect_attrs->nsections = 0;
1144 sattr = &sect_attrs->attrs[0]; 1185 sattr = &sect_attrs->attrs[0];
1145 gattr = &sect_attrs->grp.attrs[0]; 1186 gattr = &sect_attrs->grp.attrs[0];
1146 for (i = 0; i < nsect; i++) { 1187 for (i = 0; i < info->hdr->e_shnum; i++) {
1147 if (sect_empty(&sechdrs[i])) 1188 Elf_Shdr *sec = &info->sechdrs[i];
1189 if (sect_empty(sec))
1148 continue; 1190 continue;
1149 sattr->address = sechdrs[i].sh_addr; 1191 sattr->address = sec->sh_addr;
1150 sattr->name = kstrdup(secstrings + sechdrs[i].sh_name, 1192 sattr->name = kstrdup(info->secstrings + sec->sh_name,
1151 GFP_KERNEL); 1193 GFP_KERNEL);
1152 if (sattr->name == NULL) 1194 if (sattr->name == NULL)
1153 goto out; 1195 goto out;
@@ -1192,7 +1234,7 @@ struct module_notes_attrs {
1192 struct bin_attribute attrs[0]; 1234 struct bin_attribute attrs[0];
1193}; 1235};
1194 1236
1195static ssize_t module_notes_read(struct kobject *kobj, 1237static ssize_t module_notes_read(struct file *filp, struct kobject *kobj,
1196 struct bin_attribute *bin_attr, 1238 struct bin_attribute *bin_attr,
1197 char *buf, loff_t pos, size_t count) 1239 char *buf, loff_t pos, size_t count)
1198{ 1240{
@@ -1215,8 +1257,7 @@ static void free_notes_attrs(struct module_notes_attrs *notes_attrs,
1215 kfree(notes_attrs); 1257 kfree(notes_attrs);
1216} 1258}
1217 1259
1218static void add_notes_attrs(struct module *mod, unsigned int nsect, 1260static void add_notes_attrs(struct module *mod, const struct load_info *info)
1219 char *secstrings, Elf_Shdr *sechdrs)
1220{ 1261{
1221 unsigned int notes, loaded, i; 1262 unsigned int notes, loaded, i;
1222 struct module_notes_attrs *notes_attrs; 1263 struct module_notes_attrs *notes_attrs;
@@ -1228,9 +1269,9 @@ static void add_notes_attrs(struct module *mod, unsigned int nsect,
1228 1269
1229 /* Count notes sections and allocate structures. */ 1270 /* Count notes sections and allocate structures. */
1230 notes = 0; 1271 notes = 0;
1231 for (i = 0; i < nsect; i++) 1272 for (i = 0; i < info->hdr->e_shnum; i++)
1232 if (!sect_empty(&sechdrs[i]) && 1273 if (!sect_empty(&info->sechdrs[i]) &&
1233 (sechdrs[i].sh_type == SHT_NOTE)) 1274 (info->sechdrs[i].sh_type == SHT_NOTE))
1234 ++notes; 1275 ++notes;
1235 1276
1236 if (notes == 0) 1277 if (notes == 0)
@@ -1244,15 +1285,15 @@ static void add_notes_attrs(struct module *mod, unsigned int nsect,
1244 1285
1245 notes_attrs->notes = notes; 1286 notes_attrs->notes = notes;
1246 nattr = &notes_attrs->attrs[0]; 1287 nattr = &notes_attrs->attrs[0];
1247 for (loaded = i = 0; i < nsect; ++i) { 1288 for (loaded = i = 0; i < info->hdr->e_shnum; ++i) {
1248 if (sect_empty(&sechdrs[i])) 1289 if (sect_empty(&info->sechdrs[i]))
1249 continue; 1290 continue;
1250 if (sechdrs[i].sh_type == SHT_NOTE) { 1291 if (info->sechdrs[i].sh_type == SHT_NOTE) {
1251 sysfs_bin_attr_init(nattr); 1292 sysfs_bin_attr_init(nattr);
1252 nattr->attr.name = mod->sect_attrs->attrs[loaded].name; 1293 nattr->attr.name = mod->sect_attrs->attrs[loaded].name;
1253 nattr->attr.mode = S_IRUGO; 1294 nattr->attr.mode = S_IRUGO;
1254 nattr->size = sechdrs[i].sh_size; 1295 nattr->size = info->sechdrs[i].sh_size;
1255 nattr->private = (void *) sechdrs[i].sh_addr; 1296 nattr->private = (void *) info->sechdrs[i].sh_addr;
1256 nattr->read = module_notes_read; 1297 nattr->read = module_notes_read;
1257 ++nattr; 1298 ++nattr;
1258 } 1299 }
@@ -1283,8 +1324,8 @@ static void remove_notes_attrs(struct module *mod)
1283 1324
1284#else 1325#else
1285 1326
1286static inline void add_sect_attrs(struct module *mod, unsigned int nsect, 1327static inline void add_sect_attrs(struct module *mod,
1287 char *sectstrings, Elf_Shdr *sechdrs) 1328 const struct load_info *info)
1288{ 1329{
1289} 1330}
1290 1331
@@ -1292,18 +1333,44 @@ static inline void remove_sect_attrs(struct module *mod)
1292{ 1333{
1293} 1334}
1294 1335
1295static inline void add_notes_attrs(struct module *mod, unsigned int nsect, 1336static inline void add_notes_attrs(struct module *mod,
1296 char *sectstrings, Elf_Shdr *sechdrs) 1337 const struct load_info *info)
1297{ 1338{
1298} 1339}
1299 1340
1300static inline void remove_notes_attrs(struct module *mod) 1341static inline void remove_notes_attrs(struct module *mod)
1301{ 1342{
1302} 1343}
1344#endif /* CONFIG_KALLSYMS */
1345
1346static void add_usage_links(struct module *mod)
1347{
1348#ifdef CONFIG_MODULE_UNLOAD
1349 struct module_use *use;
1350 int nowarn;
1351
1352 mutex_lock(&module_mutex);
1353 list_for_each_entry(use, &mod->target_list, target_list) {
1354 nowarn = sysfs_create_link(use->target->holders_dir,
1355 &mod->mkobj.kobj, mod->name);
1356 }
1357 mutex_unlock(&module_mutex);
1303#endif 1358#endif
1359}
1304 1360
1305#ifdef CONFIG_SYSFS 1361static void del_usage_links(struct module *mod)
1306int module_add_modinfo_attrs(struct module *mod) 1362{
1363#ifdef CONFIG_MODULE_UNLOAD
1364 struct module_use *use;
1365
1366 mutex_lock(&module_mutex);
1367 list_for_each_entry(use, &mod->target_list, target_list)
1368 sysfs_remove_link(use->target->holders_dir, mod->name);
1369 mutex_unlock(&module_mutex);
1370#endif
1371}
1372
1373static int module_add_modinfo_attrs(struct module *mod)
1307{ 1374{
1308 struct module_attribute *attr; 1375 struct module_attribute *attr;
1309 struct module_attribute *temp_attr; 1376 struct module_attribute *temp_attr;
@@ -1329,7 +1396,7 @@ int module_add_modinfo_attrs(struct module *mod)
1329 return error; 1396 return error;
1330} 1397}
1331 1398
1332void module_remove_modinfo_attrs(struct module *mod) 1399static void module_remove_modinfo_attrs(struct module *mod)
1333{ 1400{
1334 struct module_attribute *attr; 1401 struct module_attribute *attr;
1335 int i; 1402 int i;
@@ -1345,7 +1412,7 @@ void module_remove_modinfo_attrs(struct module *mod)
1345 kfree(mod->modinfo_attrs); 1412 kfree(mod->modinfo_attrs);
1346} 1413}
1347 1414
1348int mod_sysfs_init(struct module *mod) 1415static int mod_sysfs_init(struct module *mod)
1349{ 1416{
1350 int err; 1417 int err;
1351 struct kobject *kobj; 1418 struct kobject *kobj;
@@ -1379,12 +1446,17 @@ out:
1379 return err; 1446 return err;
1380} 1447}
1381 1448
1382int mod_sysfs_setup(struct module *mod, 1449static int mod_sysfs_setup(struct module *mod,
1450 const struct load_info *info,
1383 struct kernel_param *kparam, 1451 struct kernel_param *kparam,
1384 unsigned int num_params) 1452 unsigned int num_params)
1385{ 1453{
1386 int err; 1454 int err;
1387 1455
1456 err = mod_sysfs_init(mod);
1457 if (err)
1458 goto out;
1459
1388 mod->holders_dir = kobject_create_and_add("holders", &mod->mkobj.kobj); 1460 mod->holders_dir = kobject_create_and_add("holders", &mod->mkobj.kobj);
1389 if (!mod->holders_dir) { 1461 if (!mod->holders_dir) {
1390 err = -ENOMEM; 1462 err = -ENOMEM;
@@ -1399,6 +1471,10 @@ int mod_sysfs_setup(struct module *mod,
1399 if (err) 1471 if (err)
1400 goto out_unreg_param; 1472 goto out_unreg_param;
1401 1473
1474 add_usage_links(mod);
1475 add_sect_attrs(mod, info);
1476 add_notes_attrs(mod, info);
1477
1402 kobject_uevent(&mod->mkobj.kobj, KOBJ_ADD); 1478 kobject_uevent(&mod->mkobj.kobj, KOBJ_ADD);
1403 return 0; 1479 return 0;
1404 1480
@@ -1408,24 +1484,44 @@ out_unreg_holders:
1408 kobject_put(mod->holders_dir); 1484 kobject_put(mod->holders_dir);
1409out_unreg: 1485out_unreg:
1410 kobject_put(&mod->mkobj.kobj); 1486 kobject_put(&mod->mkobj.kobj);
1487out:
1411 return err; 1488 return err;
1412} 1489}
1413 1490
1414static void mod_sysfs_fini(struct module *mod) 1491static void mod_sysfs_fini(struct module *mod)
1415{ 1492{
1493 remove_notes_attrs(mod);
1494 remove_sect_attrs(mod);
1416 kobject_put(&mod->mkobj.kobj); 1495 kobject_put(&mod->mkobj.kobj);
1417} 1496}
1418 1497
1419#else /* CONFIG_SYSFS */ 1498#else /* !CONFIG_SYSFS */
1499
1500static int mod_sysfs_setup(struct module *mod,
1501 const struct load_info *info,
1502 struct kernel_param *kparam,
1503 unsigned int num_params)
1504{
1505 return 0;
1506}
1420 1507
1421static void mod_sysfs_fini(struct module *mod) 1508static void mod_sysfs_fini(struct module *mod)
1422{ 1509{
1423} 1510}
1424 1511
1512static void module_remove_modinfo_attrs(struct module *mod)
1513{
1514}
1515
1516static void del_usage_links(struct module *mod)
1517{
1518}
1519
1425#endif /* CONFIG_SYSFS */ 1520#endif /* CONFIG_SYSFS */
1426 1521
1427static void mod_kobject_remove(struct module *mod) 1522static void mod_sysfs_teardown(struct module *mod)
1428{ 1523{
1524 del_usage_links(mod);
1429 module_remove_modinfo_attrs(mod); 1525 module_remove_modinfo_attrs(mod);
1430 module_param_sysfs_remove(mod); 1526 module_param_sysfs_remove(mod);
1431 kobject_put(mod->mkobj.drivers_dir); 1527 kobject_put(mod->mkobj.drivers_dir);
@@ -1444,16 +1540,19 @@ static int __unlink_module(void *_mod)
1444 return 0; 1540 return 0;
1445} 1541}
1446 1542
1447/* Free a module, remove from lists, etc (must hold module_mutex). */ 1543/* Free a module, remove from lists, etc. */
1448static void free_module(struct module *mod) 1544static void free_module(struct module *mod)
1449{ 1545{
1450 trace_module_free(mod); 1546 trace_module_free(mod);
1451 1547
1452 /* Delete from various lists */ 1548 /* Delete from various lists */
1549 mutex_lock(&module_mutex);
1453 stop_machine(__unlink_module, mod, NULL); 1550 stop_machine(__unlink_module, mod, NULL);
1454 remove_notes_attrs(mod); 1551 mutex_unlock(&module_mutex);
1455 remove_sect_attrs(mod); 1552 mod_sysfs_teardown(mod);
1456 mod_kobject_remove(mod); 1553
1554 /* Remove dynamic debug info */
1555 ddebug_remove_module(mod->name);
1457 1556
1458 /* Arch-specific cleanup. */ 1557 /* Arch-specific cleanup. */
1459 module_arch_cleanup(mod); 1558 module_arch_cleanup(mod);
@@ -1468,10 +1567,7 @@ static void free_module(struct module *mod)
1468 module_free(mod, mod->module_init); 1567 module_free(mod, mod->module_init);
1469 kfree(mod->args); 1568 kfree(mod->args);
1470 percpu_modfree(mod); 1569 percpu_modfree(mod);
1471#if defined(CONFIG_MODULE_UNLOAD) 1570
1472 if (mod->refptr)
1473 free_percpu(mod->refptr);
1474#endif
1475 /* Free lock-classes: */ 1571 /* Free lock-classes: */
1476 lockdep_free_key_range(mod->module_core, mod->core_size); 1572 lockdep_free_key_range(mod->module_core, mod->core_size);
1477 1573
@@ -1501,6 +1597,8 @@ EXPORT_SYMBOL_GPL(__symbol_get);
1501/* 1597/*
1502 * Ensure that an exported symbol [global namespace] does not already exist 1598 * Ensure that an exported symbol [global namespace] does not already exist
1503 * in the kernel or in some other module's exported symbol table. 1599 * in the kernel or in some other module's exported symbol table.
1600 *
1601 * You must hold the module_mutex.
1504 */ 1602 */
1505static int verify_export_symbols(struct module *mod) 1603static int verify_export_symbols(struct module *mod)
1506{ 1604{
@@ -1535,25 +1633,23 @@ static int verify_export_symbols(struct module *mod)
1535} 1633}
1536 1634
1537/* Change all symbols so that st_value encodes the pointer directly. */ 1635/* Change all symbols so that st_value encodes the pointer directly. */
1538static int simplify_symbols(Elf_Shdr *sechdrs, 1636static int simplify_symbols(struct module *mod, const struct load_info *info)
1539 unsigned int symindex, 1637{
1540 const char *strtab, 1638 Elf_Shdr *symsec = &info->sechdrs[info->index.sym];
1541 unsigned int versindex, 1639 Elf_Sym *sym = (void *)symsec->sh_addr;
1542 unsigned int pcpuindex,
1543 struct module *mod)
1544{
1545 Elf_Sym *sym = (void *)sechdrs[symindex].sh_addr;
1546 unsigned long secbase; 1640 unsigned long secbase;
1547 unsigned int i, n = sechdrs[symindex].sh_size / sizeof(Elf_Sym); 1641 unsigned int i;
1548 int ret = 0; 1642 int ret = 0;
1549 const struct kernel_symbol *ksym; 1643 const struct kernel_symbol *ksym;
1550 1644
1551 for (i = 1; i < n; i++) { 1645 for (i = 1; i < symsec->sh_size / sizeof(Elf_Sym); i++) {
1646 const char *name = info->strtab + sym[i].st_name;
1647
1552 switch (sym[i].st_shndx) { 1648 switch (sym[i].st_shndx) {
1553 case SHN_COMMON: 1649 case SHN_COMMON:
1554 /* We compiled with -fno-common. These are not 1650 /* We compiled with -fno-common. These are not
1555 supposed to happen. */ 1651 supposed to happen. */
1556 DEBUGP("Common symbol: %s\n", strtab + sym[i].st_name); 1652 DEBUGP("Common symbol: %s\n", name);
1557 printk("%s: please compile with -fno-common\n", 1653 printk("%s: please compile with -fno-common\n",
1558 mod->name); 1654 mod->name);
1559 ret = -ENOEXEC; 1655 ret = -ENOEXEC;
@@ -1566,29 +1662,28 @@ static int simplify_symbols(Elf_Shdr *sechdrs,
1566 break; 1662 break;
1567 1663
1568 case SHN_UNDEF: 1664 case SHN_UNDEF:
1569 ksym = resolve_symbol(sechdrs, versindex, 1665 ksym = resolve_symbol_wait(mod, info, name);
1570 strtab + sym[i].st_name, mod);
1571 /* Ok if resolved. */ 1666 /* Ok if resolved. */
1572 if (ksym) { 1667 if (ksym && !IS_ERR(ksym)) {
1573 sym[i].st_value = ksym->value; 1668 sym[i].st_value = ksym->value;
1574 break; 1669 break;
1575 } 1670 }
1576 1671
1577 /* Ok if weak. */ 1672 /* Ok if weak. */
1578 if (ELF_ST_BIND(sym[i].st_info) == STB_WEAK) 1673 if (!ksym && ELF_ST_BIND(sym[i].st_info) == STB_WEAK)
1579 break; 1674 break;
1580 1675
1581 printk(KERN_WARNING "%s: Unknown symbol %s\n", 1676 printk(KERN_WARNING "%s: Unknown symbol %s (err %li)\n",
1582 mod->name, strtab + sym[i].st_name); 1677 mod->name, name, PTR_ERR(ksym));
1583 ret = -ENOENT; 1678 ret = PTR_ERR(ksym) ?: -ENOENT;
1584 break; 1679 break;
1585 1680
1586 default: 1681 default:
1587 /* Divert to percpu allocation if a percpu var. */ 1682 /* Divert to percpu allocation if a percpu var. */
1588 if (sym[i].st_shndx == pcpuindex) 1683 if (sym[i].st_shndx == info->index.pcpu)
1589 secbase = (unsigned long)mod_percpu(mod); 1684 secbase = (unsigned long)mod_percpu(mod);
1590 else 1685 else
1591 secbase = sechdrs[sym[i].st_shndx].sh_addr; 1686 secbase = info->sechdrs[sym[i].st_shndx].sh_addr;
1592 sym[i].st_value += secbase; 1687 sym[i].st_value += secbase;
1593 break; 1688 break;
1594 } 1689 }
@@ -1597,6 +1692,35 @@ static int simplify_symbols(Elf_Shdr *sechdrs,
1597 return ret; 1692 return ret;
1598} 1693}
1599 1694
1695static int apply_relocations(struct module *mod, const struct load_info *info)
1696{
1697 unsigned int i;
1698 int err = 0;
1699
1700 /* Now do relocations. */
1701 for (i = 1; i < info->hdr->e_shnum; i++) {
1702 unsigned int infosec = info->sechdrs[i].sh_info;
1703
1704 /* Not a valid relocation section? */
1705 if (infosec >= info->hdr->e_shnum)
1706 continue;
1707
1708 /* Don't bother with non-allocated sections */
1709 if (!(info->sechdrs[infosec].sh_flags & SHF_ALLOC))
1710 continue;
1711
1712 if (info->sechdrs[i].sh_type == SHT_REL)
1713 err = apply_relocate(info->sechdrs, info->strtab,
1714 info->index.sym, i, mod);
1715 else if (info->sechdrs[i].sh_type == SHT_RELA)
1716 err = apply_relocate_add(info->sechdrs, info->strtab,
1717 info->index.sym, i, mod);
1718 if (err < 0)
1719 break;
1720 }
1721 return err;
1722}
1723
1600/* Additional bytes needed by arch in front of individual sections */ 1724/* Additional bytes needed by arch in front of individual sections */
1601unsigned int __weak arch_mod_section_prepend(struct module *mod, 1725unsigned int __weak arch_mod_section_prepend(struct module *mod,
1602 unsigned int section) 1726 unsigned int section)
@@ -1621,10 +1745,7 @@ static long get_offset(struct module *mod, unsigned int *size,
1621 might -- code, read-only data, read-write data, small data. Tally 1745 might -- code, read-only data, read-write data, small data. Tally
1622 sizes, and place the offsets into sh_entsize fields: high bit means it 1746 sizes, and place the offsets into sh_entsize fields: high bit means it
1623 belongs in init. */ 1747 belongs in init. */
1624static void layout_sections(struct module *mod, 1748static void layout_sections(struct module *mod, struct load_info *info)
1625 const Elf_Ehdr *hdr,
1626 Elf_Shdr *sechdrs,
1627 const char *secstrings)
1628{ 1749{
1629 static unsigned long const masks[][2] = { 1750 static unsigned long const masks[][2] = {
1630 /* NOTE: all executable code must be the first section 1751 /* NOTE: all executable code must be the first section
@@ -1637,21 +1758,22 @@ static void layout_sections(struct module *mod,
1637 }; 1758 };
1638 unsigned int m, i; 1759 unsigned int m, i;
1639 1760
1640 for (i = 0; i < hdr->e_shnum; i++) 1761 for (i = 0; i < info->hdr->e_shnum; i++)
1641 sechdrs[i].sh_entsize = ~0UL; 1762 info->sechdrs[i].sh_entsize = ~0UL;
1642 1763
1643 DEBUGP("Core section allocation order:\n"); 1764 DEBUGP("Core section allocation order:\n");
1644 for (m = 0; m < ARRAY_SIZE(masks); ++m) { 1765 for (m = 0; m < ARRAY_SIZE(masks); ++m) {
1645 for (i = 0; i < hdr->e_shnum; ++i) { 1766 for (i = 0; i < info->hdr->e_shnum; ++i) {
1646 Elf_Shdr *s = &sechdrs[i]; 1767 Elf_Shdr *s = &info->sechdrs[i];
1768 const char *sname = info->secstrings + s->sh_name;
1647 1769
1648 if ((s->sh_flags & masks[m][0]) != masks[m][0] 1770 if ((s->sh_flags & masks[m][0]) != masks[m][0]
1649 || (s->sh_flags & masks[m][1]) 1771 || (s->sh_flags & masks[m][1])
1650 || s->sh_entsize != ~0UL 1772 || s->sh_entsize != ~0UL
1651 || strstarts(secstrings + s->sh_name, ".init")) 1773 || strstarts(sname, ".init"))
1652 continue; 1774 continue;
1653 s->sh_entsize = get_offset(mod, &mod->core_size, s, i); 1775 s->sh_entsize = get_offset(mod, &mod->core_size, s, i);
1654 DEBUGP("\t%s\n", secstrings + s->sh_name); 1776 DEBUGP("\t%s\n", name);
1655 } 1777 }
1656 if (m == 0) 1778 if (m == 0)
1657 mod->core_text_size = mod->core_size; 1779 mod->core_text_size = mod->core_size;
@@ -1659,17 +1781,18 @@ static void layout_sections(struct module *mod,
1659 1781
1660 DEBUGP("Init section allocation order:\n"); 1782 DEBUGP("Init section allocation order:\n");
1661 for (m = 0; m < ARRAY_SIZE(masks); ++m) { 1783 for (m = 0; m < ARRAY_SIZE(masks); ++m) {
1662 for (i = 0; i < hdr->e_shnum; ++i) { 1784 for (i = 0; i < info->hdr->e_shnum; ++i) {
1663 Elf_Shdr *s = &sechdrs[i]; 1785 Elf_Shdr *s = &info->sechdrs[i];
1786 const char *sname = info->secstrings + s->sh_name;
1664 1787
1665 if ((s->sh_flags & masks[m][0]) != masks[m][0] 1788 if ((s->sh_flags & masks[m][0]) != masks[m][0]
1666 || (s->sh_flags & masks[m][1]) 1789 || (s->sh_flags & masks[m][1])
1667 || s->sh_entsize != ~0UL 1790 || s->sh_entsize != ~0UL
1668 || !strstarts(secstrings + s->sh_name, ".init")) 1791 || !strstarts(sname, ".init"))
1669 continue; 1792 continue;
1670 s->sh_entsize = (get_offset(mod, &mod->init_size, s, i) 1793 s->sh_entsize = (get_offset(mod, &mod->init_size, s, i)
1671 | INIT_OFFSET_MASK); 1794 | INIT_OFFSET_MASK);
1672 DEBUGP("\t%s\n", secstrings + s->sh_name); 1795 DEBUGP("\t%s\n", sname);
1673 } 1796 }
1674 if (m == 0) 1797 if (m == 0)
1675 mod->init_text_size = mod->init_size; 1798 mod->init_text_size = mod->init_size;
@@ -1708,33 +1831,28 @@ static char *next_string(char *string, unsigned long *secsize)
1708 return string; 1831 return string;
1709} 1832}
1710 1833
1711static char *get_modinfo(Elf_Shdr *sechdrs, 1834static char *get_modinfo(struct load_info *info, const char *tag)
1712 unsigned int info,
1713 const char *tag)
1714{ 1835{
1715 char *p; 1836 char *p;
1716 unsigned int taglen = strlen(tag); 1837 unsigned int taglen = strlen(tag);
1717 unsigned long size = sechdrs[info].sh_size; 1838 Elf_Shdr *infosec = &info->sechdrs[info->index.info];
1839 unsigned long size = infosec->sh_size;
1718 1840
1719 for (p = (char *)sechdrs[info].sh_addr; p; p = next_string(p, &size)) { 1841 for (p = (char *)infosec->sh_addr; p; p = next_string(p, &size)) {
1720 if (strncmp(p, tag, taglen) == 0 && p[taglen] == '=') 1842 if (strncmp(p, tag, taglen) == 0 && p[taglen] == '=')
1721 return p + taglen + 1; 1843 return p + taglen + 1;
1722 } 1844 }
1723 return NULL; 1845 return NULL;
1724} 1846}
1725 1847
1726static void setup_modinfo(struct module *mod, Elf_Shdr *sechdrs, 1848static void setup_modinfo(struct module *mod, struct load_info *info)
1727 unsigned int infoindex)
1728{ 1849{
1729 struct module_attribute *attr; 1850 struct module_attribute *attr;
1730 int i; 1851 int i;
1731 1852
1732 for (i = 0; (attr = modinfo_attrs[i]); i++) { 1853 for (i = 0; (attr = modinfo_attrs[i]); i++) {
1733 if (attr->setup) 1854 if (attr->setup)
1734 attr->setup(mod, 1855 attr->setup(mod, get_modinfo(info, attr->attr.name));
1735 get_modinfo(sechdrs,
1736 infoindex,
1737 attr->attr.name));
1738 } 1856 }
1739} 1857}
1740 1858
@@ -1775,11 +1893,10 @@ static int is_exported(const char *name, unsigned long value,
1775} 1893}
1776 1894
1777/* As per nm */ 1895/* As per nm */
1778static char elf_type(const Elf_Sym *sym, 1896static char elf_type(const Elf_Sym *sym, const struct load_info *info)
1779 Elf_Shdr *sechdrs,
1780 const char *secstrings,
1781 struct module *mod)
1782{ 1897{
1898 const Elf_Shdr *sechdrs = info->sechdrs;
1899
1783 if (ELF_ST_BIND(sym->st_info) == STB_WEAK) { 1900 if (ELF_ST_BIND(sym->st_info) == STB_WEAK) {
1784 if (ELF_ST_TYPE(sym->st_info) == STT_OBJECT) 1901 if (ELF_ST_TYPE(sym->st_info) == STT_OBJECT)
1785 return 'v'; 1902 return 'v';
@@ -1809,8 +1926,10 @@ static char elf_type(const Elf_Sym *sym,
1809 else 1926 else
1810 return 'b'; 1927 return 'b';
1811 } 1928 }
1812 if (strstarts(secstrings + sechdrs[sym->st_shndx].sh_name, ".debug")) 1929 if (strstarts(info->secstrings + sechdrs[sym->st_shndx].sh_name,
1930 ".debug")) {
1813 return 'n'; 1931 return 'n';
1932 }
1814 return '?'; 1933 return '?';
1815} 1934}
1816 1935
@@ -1835,127 +1954,96 @@ static bool is_core_symbol(const Elf_Sym *src, const Elf_Shdr *sechdrs,
1835 return true; 1954 return true;
1836} 1955}
1837 1956
1838static unsigned long layout_symtab(struct module *mod, 1957static void layout_symtab(struct module *mod, struct load_info *info)
1839 Elf_Shdr *sechdrs,
1840 unsigned int symindex,
1841 unsigned int strindex,
1842 const Elf_Ehdr *hdr,
1843 const char *secstrings,
1844 unsigned long *pstroffs,
1845 unsigned long *strmap)
1846{ 1958{
1847 unsigned long symoffs; 1959 Elf_Shdr *symsect = info->sechdrs + info->index.sym;
1848 Elf_Shdr *symsect = sechdrs + symindex; 1960 Elf_Shdr *strsect = info->sechdrs + info->index.str;
1849 Elf_Shdr *strsect = sechdrs + strindex;
1850 const Elf_Sym *src; 1961 const Elf_Sym *src;
1851 const char *strtab;
1852 unsigned int i, nsrc, ndst; 1962 unsigned int i, nsrc, ndst;
1853 1963
1854 /* Put symbol section at end of init part of module. */ 1964 /* Put symbol section at end of init part of module. */
1855 symsect->sh_flags |= SHF_ALLOC; 1965 symsect->sh_flags |= SHF_ALLOC;
1856 symsect->sh_entsize = get_offset(mod, &mod->init_size, symsect, 1966 symsect->sh_entsize = get_offset(mod, &mod->init_size, symsect,
1857 symindex) | INIT_OFFSET_MASK; 1967 info->index.sym) | INIT_OFFSET_MASK;
1858 DEBUGP("\t%s\n", secstrings + symsect->sh_name); 1968 DEBUGP("\t%s\n", info->secstrings + symsect->sh_name);
1859 1969
1860 src = (void *)hdr + symsect->sh_offset; 1970 src = (void *)info->hdr + symsect->sh_offset;
1861 nsrc = symsect->sh_size / sizeof(*src); 1971 nsrc = symsect->sh_size / sizeof(*src);
1862 strtab = (void *)hdr + strsect->sh_offset;
1863 for (ndst = i = 1; i < nsrc; ++i, ++src) 1972 for (ndst = i = 1; i < nsrc; ++i, ++src)
1864 if (is_core_symbol(src, sechdrs, hdr->e_shnum)) { 1973 if (is_core_symbol(src, info->sechdrs, info->hdr->e_shnum)) {
1865 unsigned int j = src->st_name; 1974 unsigned int j = src->st_name;
1866 1975
1867 while(!__test_and_set_bit(j, strmap) && strtab[j]) 1976 while (!__test_and_set_bit(j, info->strmap)
1977 && info->strtab[j])
1868 ++j; 1978 ++j;
1869 ++ndst; 1979 ++ndst;
1870 } 1980 }
1871 1981
1872 /* Append room for core symbols at end of core part. */ 1982 /* Append room for core symbols at end of core part. */
1873 symoffs = ALIGN(mod->core_size, symsect->sh_addralign ?: 1); 1983 info->symoffs = ALIGN(mod->core_size, symsect->sh_addralign ?: 1);
1874 mod->core_size = symoffs + ndst * sizeof(Elf_Sym); 1984 mod->core_size = info->symoffs + ndst * sizeof(Elf_Sym);
1875 1985
1876 /* Put string table section at end of init part of module. */ 1986 /* Put string table section at end of init part of module. */
1877 strsect->sh_flags |= SHF_ALLOC; 1987 strsect->sh_flags |= SHF_ALLOC;
1878 strsect->sh_entsize = get_offset(mod, &mod->init_size, strsect, 1988 strsect->sh_entsize = get_offset(mod, &mod->init_size, strsect,
1879 strindex) | INIT_OFFSET_MASK; 1989 info->index.str) | INIT_OFFSET_MASK;
1880 DEBUGP("\t%s\n", secstrings + strsect->sh_name); 1990 DEBUGP("\t%s\n", info->secstrings + strsect->sh_name);
1881 1991
1882 /* Append room for core symbols' strings at end of core part. */ 1992 /* Append room for core symbols' strings at end of core part. */
1883 *pstroffs = mod->core_size; 1993 info->stroffs = mod->core_size;
1884 __set_bit(0, strmap); 1994 __set_bit(0, info->strmap);
1885 mod->core_size += bitmap_weight(strmap, strsect->sh_size); 1995 mod->core_size += bitmap_weight(info->strmap, strsect->sh_size);
1886
1887 return symoffs;
1888} 1996}
1889 1997
1890static void add_kallsyms(struct module *mod, 1998static void add_kallsyms(struct module *mod, const struct load_info *info)
1891 Elf_Shdr *sechdrs,
1892 unsigned int shnum,
1893 unsigned int symindex,
1894 unsigned int strindex,
1895 unsigned long symoffs,
1896 unsigned long stroffs,
1897 const char *secstrings,
1898 unsigned long *strmap)
1899{ 1999{
1900 unsigned int i, ndst; 2000 unsigned int i, ndst;
1901 const Elf_Sym *src; 2001 const Elf_Sym *src;
1902 Elf_Sym *dst; 2002 Elf_Sym *dst;
1903 char *s; 2003 char *s;
2004 Elf_Shdr *symsec = &info->sechdrs[info->index.sym];
1904 2005
1905 mod->symtab = (void *)sechdrs[symindex].sh_addr; 2006 mod->symtab = (void *)symsec->sh_addr;
1906 mod->num_symtab = sechdrs[symindex].sh_size / sizeof(Elf_Sym); 2007 mod->num_symtab = symsec->sh_size / sizeof(Elf_Sym);
1907 mod->strtab = (void *)sechdrs[strindex].sh_addr; 2008 /* Make sure we get permanent strtab: don't use info->strtab. */
2009 mod->strtab = (void *)info->sechdrs[info->index.str].sh_addr;
1908 2010
1909 /* Set types up while we still have access to sections. */ 2011 /* Set types up while we still have access to sections. */
1910 for (i = 0; i < mod->num_symtab; i++) 2012 for (i = 0; i < mod->num_symtab; i++)
1911 mod->symtab[i].st_info 2013 mod->symtab[i].st_info = elf_type(&mod->symtab[i], info);
1912 = elf_type(&mod->symtab[i], sechdrs, secstrings, mod);
1913 2014
1914 mod->core_symtab = dst = mod->module_core + symoffs; 2015 mod->core_symtab = dst = mod->module_core + info->symoffs;
1915 src = mod->symtab; 2016 src = mod->symtab;
1916 *dst = *src; 2017 *dst = *src;
1917 for (ndst = i = 1; i < mod->num_symtab; ++i, ++src) { 2018 for (ndst = i = 1; i < mod->num_symtab; ++i, ++src) {
1918 if (!is_core_symbol(src, sechdrs, shnum)) 2019 if (!is_core_symbol(src, info->sechdrs, info->hdr->e_shnum))
1919 continue; 2020 continue;
1920 dst[ndst] = *src; 2021 dst[ndst] = *src;
1921 dst[ndst].st_name = bitmap_weight(strmap, dst[ndst].st_name); 2022 dst[ndst].st_name = bitmap_weight(info->strmap,
2023 dst[ndst].st_name);
1922 ++ndst; 2024 ++ndst;
1923 } 2025 }
1924 mod->core_num_syms = ndst; 2026 mod->core_num_syms = ndst;
1925 2027
1926 mod->core_strtab = s = mod->module_core + stroffs; 2028 mod->core_strtab = s = mod->module_core + info->stroffs;
1927 for (*s = 0, i = 1; i < sechdrs[strindex].sh_size; ++i) 2029 for (*s = 0, i = 1; i < info->sechdrs[info->index.str].sh_size; ++i)
1928 if (test_bit(i, strmap)) 2030 if (test_bit(i, info->strmap))
1929 *++s = mod->strtab[i]; 2031 *++s = mod->strtab[i];
1930} 2032}
1931#else 2033#else
1932static inline unsigned long layout_symtab(struct module *mod, 2034static inline void layout_symtab(struct module *mod, struct load_info *info)
1933 Elf_Shdr *sechdrs,
1934 unsigned int symindex,
1935 unsigned int strindex,
1936 const Elf_Ehdr *hdr,
1937 const char *secstrings,
1938 unsigned long *pstroffs,
1939 unsigned long *strmap)
1940{ 2035{
1941 return 0;
1942} 2036}
1943 2037
1944static inline void add_kallsyms(struct module *mod, 2038static void add_kallsyms(struct module *mod, struct load_info *info)
1945 Elf_Shdr *sechdrs,
1946 unsigned int shnum,
1947 unsigned int symindex,
1948 unsigned int strindex,
1949 unsigned long symoffs,
1950 unsigned long stroffs,
1951 const char *secstrings,
1952 const unsigned long *strmap)
1953{ 2039{
1954} 2040}
1955#endif /* CONFIG_KALLSYMS */ 2041#endif /* CONFIG_KALLSYMS */
1956 2042
1957static void dynamic_debug_setup(struct _ddebug *debug, unsigned int num) 2043static void dynamic_debug_setup(struct _ddebug *debug, unsigned int num)
1958{ 2044{
2045 if (!debug)
2046 return;
1959#ifdef CONFIG_DYNAMIC_DEBUG 2047#ifdef CONFIG_DYNAMIC_DEBUG
1960 if (ddebug_add_module(debug, num, debug->modname)) 2048 if (ddebug_add_module(debug, num, debug->modname))
1961 printk(KERN_ERR "dynamic debug error adding module: %s\n", 2049 printk(KERN_ERR "dynamic debug error adding module: %s\n",
@@ -1963,77 +2051,70 @@ static void dynamic_debug_setup(struct _ddebug *debug, unsigned int num)
1963#endif 2051#endif
1964} 2052}
1965 2053
2054static void dynamic_debug_remove(struct _ddebug *debug)
2055{
2056 if (debug)
2057 ddebug_remove_module(debug->modname);
2058}
2059
1966static void *module_alloc_update_bounds(unsigned long size) 2060static void *module_alloc_update_bounds(unsigned long size)
1967{ 2061{
1968 void *ret = module_alloc(size); 2062 void *ret = module_alloc(size);
1969 2063
1970 if (ret) { 2064 if (ret) {
2065 mutex_lock(&module_mutex);
1971 /* Update module bounds. */ 2066 /* Update module bounds. */
1972 if ((unsigned long)ret < module_addr_min) 2067 if ((unsigned long)ret < module_addr_min)
1973 module_addr_min = (unsigned long)ret; 2068 module_addr_min = (unsigned long)ret;
1974 if ((unsigned long)ret + size > module_addr_max) 2069 if ((unsigned long)ret + size > module_addr_max)
1975 module_addr_max = (unsigned long)ret + size; 2070 module_addr_max = (unsigned long)ret + size;
2071 mutex_unlock(&module_mutex);
1976 } 2072 }
1977 return ret; 2073 return ret;
1978} 2074}
1979 2075
1980#ifdef CONFIG_DEBUG_KMEMLEAK 2076#ifdef CONFIG_DEBUG_KMEMLEAK
1981static void kmemleak_load_module(struct module *mod, Elf_Ehdr *hdr, 2077static void kmemleak_load_module(const struct module *mod,
1982 Elf_Shdr *sechdrs, char *secstrings) 2078 const struct load_info *info)
1983{ 2079{
1984 unsigned int i; 2080 unsigned int i;
1985 2081
1986 /* only scan the sections containing data */ 2082 /* only scan the sections containing data */
1987 kmemleak_scan_area(mod, sizeof(struct module), GFP_KERNEL); 2083 kmemleak_scan_area(mod, sizeof(struct module), GFP_KERNEL);
1988 2084
1989 for (i = 1; i < hdr->e_shnum; i++) { 2085 for (i = 1; i < info->hdr->e_shnum; i++) {
1990 if (!(sechdrs[i].sh_flags & SHF_ALLOC)) 2086 const char *name = info->secstrings + info->sechdrs[i].sh_name;
2087 if (!(info->sechdrs[i].sh_flags & SHF_ALLOC))
1991 continue; 2088 continue;
1992 if (strncmp(secstrings + sechdrs[i].sh_name, ".data", 5) != 0 2089 if (!strstarts(name, ".data") && !strstarts(name, ".bss"))
1993 && strncmp(secstrings + sechdrs[i].sh_name, ".bss", 4) != 0)
1994 continue; 2090 continue;
1995 2091
1996 kmemleak_scan_area((void *)sechdrs[i].sh_addr, 2092 kmemleak_scan_area((void *)info->sechdrs[i].sh_addr,
1997 sechdrs[i].sh_size, GFP_KERNEL); 2093 info->sechdrs[i].sh_size, GFP_KERNEL);
1998 } 2094 }
1999} 2095}
2000#else 2096#else
2001static inline void kmemleak_load_module(struct module *mod, Elf_Ehdr *hdr, 2097static inline void kmemleak_load_module(const struct module *mod,
2002 Elf_Shdr *sechdrs, char *secstrings) 2098 const struct load_info *info)
2003{ 2099{
2004} 2100}
2005#endif 2101#endif
2006 2102
2007/* Allocate and load the module: note that size of section 0 is always 2103/* Sets info->hdr and info->len. */
2008 zero, and we rely on this for optional sections. */ 2104static int copy_and_check(struct load_info *info,
2009static noinline struct module *load_module(void __user *umod, 2105 const void __user *umod, unsigned long len,
2010 unsigned long len, 2106 const char __user *uargs)
2011 const char __user *uargs)
2012{ 2107{
2108 int err;
2013 Elf_Ehdr *hdr; 2109 Elf_Ehdr *hdr;
2014 Elf_Shdr *sechdrs;
2015 char *secstrings, *args, *modmagic, *strtab = NULL;
2016 char *staging;
2017 unsigned int i;
2018 unsigned int symindex = 0;
2019 unsigned int strindex = 0;
2020 unsigned int modindex, versindex, infoindex, pcpuindex;
2021 struct module *mod;
2022 long err = 0;
2023 void *ptr = NULL; /* Stops spurious gcc warning */
2024 unsigned long symoffs, stroffs, *strmap;
2025
2026 mm_segment_t old_fs;
2027 2110
2028 DEBUGP("load_module: umod=%p, len=%lu, uargs=%p\n",
2029 umod, len, uargs);
2030 if (len < sizeof(*hdr)) 2111 if (len < sizeof(*hdr))
2031 return ERR_PTR(-ENOEXEC); 2112 return -ENOEXEC;
2032 2113
2033 /* Suck in entire file: we'll want most of it. */ 2114 /* Suck in entire file: we'll want most of it. */
2034 /* vmalloc barfs on "unusual" numbers. Check here */ 2115 /* vmalloc barfs on "unusual" numbers. Check here */
2035 if (len > 64 * 1024 * 1024 || (hdr = vmalloc(len)) == NULL) 2116 if (len > 64 * 1024 * 1024 || (hdr = vmalloc(len)) == NULL)
2036 return ERR_PTR(-ENOMEM); 2117 return -ENOMEM;
2037 2118
2038 if (copy_from_user(hdr, umod, len) != 0) { 2119 if (copy_from_user(hdr, umod, len) != 0) {
2039 err = -EFAULT; 2120 err = -EFAULT;
@@ -2041,138 +2122,225 @@ static noinline struct module *load_module(void __user *umod,
2041 } 2122 }
2042 2123
2043 /* Sanity checks against insmoding binaries or wrong arch, 2124 /* Sanity checks against insmoding binaries or wrong arch,
2044 weird elf version */ 2125 weird elf version */
2045 if (memcmp(hdr->e_ident, ELFMAG, SELFMAG) != 0 2126 if (memcmp(hdr->e_ident, ELFMAG, SELFMAG) != 0
2046 || hdr->e_type != ET_REL 2127 || hdr->e_type != ET_REL
2047 || !elf_check_arch(hdr) 2128 || !elf_check_arch(hdr)
2048 || hdr->e_shentsize != sizeof(*sechdrs)) { 2129 || hdr->e_shentsize != sizeof(Elf_Shdr)) {
2130 err = -ENOEXEC;
2131 goto free_hdr;
2132 }
2133
2134 if (len < hdr->e_shoff + hdr->e_shnum * sizeof(Elf_Shdr)) {
2049 err = -ENOEXEC; 2135 err = -ENOEXEC;
2050 goto free_hdr; 2136 goto free_hdr;
2051 } 2137 }
2052 2138
2053 if (len < hdr->e_shoff + hdr->e_shnum * sizeof(Elf_Shdr)) 2139 info->hdr = hdr;
2054 goto truncated; 2140 info->len = len;
2141 return 0;
2142
2143free_hdr:
2144 vfree(hdr);
2145 return err;
2146}
2147
2148static void free_copy(struct load_info *info)
2149{
2150 vfree(info->hdr);
2151}
2055 2152
2056 /* Convenience variables */ 2153static int rewrite_section_headers(struct load_info *info)
2057 sechdrs = (void *)hdr + hdr->e_shoff; 2154{
2058 secstrings = (void *)hdr + sechdrs[hdr->e_shstrndx].sh_offset; 2155 unsigned int i;
2059 sechdrs[0].sh_addr = 0;
2060 2156
2061 for (i = 1; i < hdr->e_shnum; i++) { 2157 /* This should always be true, but let's be sure. */
2062 if (sechdrs[i].sh_type != SHT_NOBITS 2158 info->sechdrs[0].sh_addr = 0;
2063 && len < sechdrs[i].sh_offset + sechdrs[i].sh_size) 2159
2064 goto truncated; 2160 for (i = 1; i < info->hdr->e_shnum; i++) {
2161 Elf_Shdr *shdr = &info->sechdrs[i];
2162 if (shdr->sh_type != SHT_NOBITS
2163 && info->len < shdr->sh_offset + shdr->sh_size) {
2164 printk(KERN_ERR "Module len %lu truncated\n",
2165 info->len);
2166 return -ENOEXEC;
2167 }
2065 2168
2066 /* Mark all sections sh_addr with their address in the 2169 /* Mark all sections sh_addr with their address in the
2067 temporary image. */ 2170 temporary image. */
2068 sechdrs[i].sh_addr = (size_t)hdr + sechdrs[i].sh_offset; 2171 shdr->sh_addr = (size_t)info->hdr + shdr->sh_offset;
2069 2172
2070 /* Internal symbols and strings. */
2071 if (sechdrs[i].sh_type == SHT_SYMTAB) {
2072 symindex = i;
2073 strindex = sechdrs[i].sh_link;
2074 strtab = (char *)hdr + sechdrs[strindex].sh_offset;
2075 }
2076#ifndef CONFIG_MODULE_UNLOAD 2173#ifndef CONFIG_MODULE_UNLOAD
2077 /* Don't load .exit sections */ 2174 /* Don't load .exit sections */
2078 if (strstarts(secstrings+sechdrs[i].sh_name, ".exit")) 2175 if (strstarts(info->secstrings+shdr->sh_name, ".exit"))
2079 sechdrs[i].sh_flags &= ~(unsigned long)SHF_ALLOC; 2176 shdr->sh_flags &= ~(unsigned long)SHF_ALLOC;
2080#endif 2177#endif
2081 } 2178 }
2082 2179
2083 modindex = find_sec(hdr, sechdrs, secstrings, 2180 /* Track but don't keep modinfo and version sections. */
2084 ".gnu.linkonce.this_module"); 2181 info->index.vers = find_sec(info, "__versions");
2085 if (!modindex) { 2182 info->index.info = find_sec(info, ".modinfo");
2183 info->sechdrs[info->index.info].sh_flags &= ~(unsigned long)SHF_ALLOC;
2184 info->sechdrs[info->index.vers].sh_flags &= ~(unsigned long)SHF_ALLOC;
2185 return 0;
2186}
2187
2188/*
2189 * Set up our basic convenience variables (pointers to section headers,
2190 * search for module section index etc), and do some basic section
2191 * verification.
2192 *
2193 * Return the temporary module pointer (we'll replace it with the final
2194 * one when we move the module sections around).
2195 */
2196static struct module *setup_load_info(struct load_info *info)
2197{
2198 unsigned int i;
2199 int err;
2200 struct module *mod;
2201
2202 /* Set up the convenience variables */
2203 info->sechdrs = (void *)info->hdr + info->hdr->e_shoff;
2204 info->secstrings = (void *)info->hdr
2205 + info->sechdrs[info->hdr->e_shstrndx].sh_offset;
2206
2207 err = rewrite_section_headers(info);
2208 if (err)
2209 return ERR_PTR(err);
2210
2211 /* Find internal symbols and strings. */
2212 for (i = 1; i < info->hdr->e_shnum; i++) {
2213 if (info->sechdrs[i].sh_type == SHT_SYMTAB) {
2214 info->index.sym = i;
2215 info->index.str = info->sechdrs[i].sh_link;
2216 info->strtab = (char *)info->hdr
2217 + info->sechdrs[info->index.str].sh_offset;
2218 break;
2219 }
2220 }
2221
2222 info->index.mod = find_sec(info, ".gnu.linkonce.this_module");
2223 if (!info->index.mod) {
2086 printk(KERN_WARNING "No module found in object\n"); 2224 printk(KERN_WARNING "No module found in object\n");
2087 err = -ENOEXEC; 2225 return ERR_PTR(-ENOEXEC);
2088 goto free_hdr;
2089 } 2226 }
2090 /* This is temporary: point mod into copy of data. */ 2227 /* This is temporary: point mod into copy of data. */
2091 mod = (void *)sechdrs[modindex].sh_addr; 2228 mod = (void *)info->sechdrs[info->index.mod].sh_addr;
2092 2229
2093 if (symindex == 0) { 2230 if (info->index.sym == 0) {
2094 printk(KERN_WARNING "%s: module has no symbols (stripped?)\n", 2231 printk(KERN_WARNING "%s: module has no symbols (stripped?)\n",
2095 mod->name); 2232 mod->name);
2096 err = -ENOEXEC; 2233 return ERR_PTR(-ENOEXEC);
2097 goto free_hdr;
2098 } 2234 }
2099 2235
2100 versindex = find_sec(hdr, sechdrs, secstrings, "__versions"); 2236 info->index.pcpu = find_pcpusec(info);
2101 infoindex = find_sec(hdr, sechdrs, secstrings, ".modinfo");
2102 pcpuindex = find_pcpusec(hdr, sechdrs, secstrings);
2103
2104 /* Don't keep modinfo and version sections. */
2105 sechdrs[infoindex].sh_flags &= ~(unsigned long)SHF_ALLOC;
2106 sechdrs[versindex].sh_flags &= ~(unsigned long)SHF_ALLOC;
2107 2237
2108 /* Check module struct version now, before we try to use module. */ 2238 /* Check module struct version now, before we try to use module. */
2109 if (!check_modstruct_version(sechdrs, versindex, mod)) { 2239 if (!check_modstruct_version(info->sechdrs, info->index.vers, mod))
2110 err = -ENOEXEC; 2240 return ERR_PTR(-ENOEXEC);
2111 goto free_hdr; 2241
2112 } 2242 return mod;
2243}
2244
2245static int check_modinfo(struct module *mod, struct load_info *info)
2246{
2247 const char *modmagic = get_modinfo(info, "vermagic");
2248 int err;
2113 2249
2114 modmagic = get_modinfo(sechdrs, infoindex, "vermagic");
2115 /* This is allowed: modprobe --force will invalidate it. */ 2250 /* This is allowed: modprobe --force will invalidate it. */
2116 if (!modmagic) { 2251 if (!modmagic) {
2117 err = try_to_force_load(mod, "bad vermagic"); 2252 err = try_to_force_load(mod, "bad vermagic");
2118 if (err) 2253 if (err)
2119 goto free_hdr; 2254 return err;
2120 } else if (!same_magic(modmagic, vermagic, versindex)) { 2255 } else if (!same_magic(modmagic, vermagic, info->index.vers)) {
2121 printk(KERN_ERR "%s: version magic '%s' should be '%s'\n", 2256 printk(KERN_ERR "%s: version magic '%s' should be '%s'\n",
2122 mod->name, modmagic, vermagic); 2257 mod->name, modmagic, vermagic);
2123 err = -ENOEXEC; 2258 return -ENOEXEC;
2124 goto free_hdr;
2125 } 2259 }
2126 2260
2127 staging = get_modinfo(sechdrs, infoindex, "staging"); 2261 if (get_modinfo(info, "staging")) {
2128 if (staging) {
2129 add_taint_module(mod, TAINT_CRAP); 2262 add_taint_module(mod, TAINT_CRAP);
2130 printk(KERN_WARNING "%s: module is from the staging directory," 2263 printk(KERN_WARNING "%s: module is from the staging directory,"
2131 " the quality is unknown, you have been warned.\n", 2264 " the quality is unknown, you have been warned.\n",
2132 mod->name); 2265 mod->name);
2133 } 2266 }
2134 2267
2135 /* Now copy in args */ 2268 /* Set up license info based on the info section */
2136 args = strndup_user(uargs, ~0UL >> 1); 2269 set_license(mod, get_modinfo(info, "license"));
2137 if (IS_ERR(args)) {
2138 err = PTR_ERR(args);
2139 goto free_hdr;
2140 }
2141 2270
2142 strmap = kzalloc(BITS_TO_LONGS(sechdrs[strindex].sh_size) 2271 return 0;
2143 * sizeof(long), GFP_KERNEL); 2272}
2144 if (!strmap) {
2145 err = -ENOMEM;
2146 goto free_mod;
2147 }
2148 2273
2149 if (find_module(mod->name)) { 2274static void find_module_sections(struct module *mod, struct load_info *info)
2150 err = -EEXIST; 2275{
2151 goto free_mod; 2276 mod->kp = section_objs(info, "__param",
2152 } 2277 sizeof(*mod->kp), &mod->num_kp);
2278 mod->syms = section_objs(info, "__ksymtab",
2279 sizeof(*mod->syms), &mod->num_syms);
2280 mod->crcs = section_addr(info, "__kcrctab");
2281 mod->gpl_syms = section_objs(info, "__ksymtab_gpl",
2282 sizeof(*mod->gpl_syms),
2283 &mod->num_gpl_syms);
2284 mod->gpl_crcs = section_addr(info, "__kcrctab_gpl");
2285 mod->gpl_future_syms = section_objs(info,
2286 "__ksymtab_gpl_future",
2287 sizeof(*mod->gpl_future_syms),
2288 &mod->num_gpl_future_syms);
2289 mod->gpl_future_crcs = section_addr(info, "__kcrctab_gpl_future");
2153 2290
2154 mod->state = MODULE_STATE_COMING; 2291#ifdef CONFIG_UNUSED_SYMBOLS
2292 mod->unused_syms = section_objs(info, "__ksymtab_unused",
2293 sizeof(*mod->unused_syms),
2294 &mod->num_unused_syms);
2295 mod->unused_crcs = section_addr(info, "__kcrctab_unused");
2296 mod->unused_gpl_syms = section_objs(info, "__ksymtab_unused_gpl",
2297 sizeof(*mod->unused_gpl_syms),
2298 &mod->num_unused_gpl_syms);
2299 mod->unused_gpl_crcs = section_addr(info, "__kcrctab_unused_gpl");
2300#endif
2301#ifdef CONFIG_CONSTRUCTORS
2302 mod->ctors = section_objs(info, ".ctors",
2303 sizeof(*mod->ctors), &mod->num_ctors);
2304#endif
2155 2305
2156 /* Allow arches to frob section contents and sizes. */ 2306#ifdef CONFIG_TRACEPOINTS
2157 err = module_frob_arch_sections(hdr, sechdrs, secstrings, mod); 2307 mod->tracepoints = section_objs(info, "__tracepoints",
2158 if (err < 0) 2308 sizeof(*mod->tracepoints),
2159 goto free_mod; 2309 &mod->num_tracepoints);
2310#endif
2311#ifdef CONFIG_EVENT_TRACING
2312 mod->trace_events = section_objs(info, "_ftrace_events",
2313 sizeof(*mod->trace_events),
2314 &mod->num_trace_events);
2315 /*
2316 * This section contains pointers to allocated objects in the trace
2317 * code and not scanning it leads to false positives.
2318 */
2319 kmemleak_scan_area(mod->trace_events, sizeof(*mod->trace_events) *
2320 mod->num_trace_events, GFP_KERNEL);
2321#endif
2322#ifdef CONFIG_FTRACE_MCOUNT_RECORD
2323 /* sechdrs[0].sh_size is always zero */
2324 mod->ftrace_callsites = section_objs(info, "__mcount_loc",
2325 sizeof(*mod->ftrace_callsites),
2326 &mod->num_ftrace_callsites);
2327#endif
2160 2328
2161 if (pcpuindex) { 2329 mod->extable = section_objs(info, "__ex_table",
2162 /* We have a special allocation for this section. */ 2330 sizeof(*mod->extable), &mod->num_exentries);
2163 err = percpu_modalloc(mod, sechdrs[pcpuindex].sh_size,
2164 sechdrs[pcpuindex].sh_addralign);
2165 if (err)
2166 goto free_mod;
2167 sechdrs[pcpuindex].sh_flags &= ~(unsigned long)SHF_ALLOC;
2168 }
2169 2331
2170 /* Determine total sizes, and put offsets in sh_entsize. For now 2332 if (section_addr(info, "__obsparm"))
2171 this is done generically; there doesn't appear to be any 2333 printk(KERN_WARNING "%s: Ignoring obsolete parameters\n",
2172 special cases for the architectures. */ 2334 mod->name);
2173 layout_sections(mod, hdr, sechdrs, secstrings); 2335
2174 symoffs = layout_symtab(mod, sechdrs, symindex, strindex, hdr, 2336 info->debug = section_objs(info, "__verbose",
2175 secstrings, &stroffs, strmap); 2337 sizeof(*info->debug), &info->num_debug);
2338}
2339
2340static int move_module(struct module *mod, struct load_info *info)
2341{
2342 int i;
2343 void *ptr;
2176 2344
2177 /* Do the allocs. */ 2345 /* Do the allocs. */
2178 ptr = module_alloc_update_bounds(mod->core_size); 2346 ptr = module_alloc_update_bounds(mod->core_size);
@@ -2182,10 +2350,9 @@ static noinline struct module *load_module(void __user *umod,
2182 * leak. 2350 * leak.
2183 */ 2351 */
2184 kmemleak_not_leak(ptr); 2352 kmemleak_not_leak(ptr);
2185 if (!ptr) { 2353 if (!ptr)
2186 err = -ENOMEM; 2354 return -ENOMEM;
2187 goto free_percpu; 2355
2188 }
2189 memset(ptr, 0, mod->core_size); 2356 memset(ptr, 0, mod->core_size);
2190 mod->module_core = ptr; 2357 mod->module_core = ptr;
2191 2358
@@ -2198,55 +2365,40 @@ static noinline struct module *load_module(void __user *umod,
2198 */ 2365 */
2199 kmemleak_ignore(ptr); 2366 kmemleak_ignore(ptr);
2200 if (!ptr && mod->init_size) { 2367 if (!ptr && mod->init_size) {
2201 err = -ENOMEM; 2368 module_free(mod, mod->module_core);
2202 goto free_core; 2369 return -ENOMEM;
2203 } 2370 }
2204 memset(ptr, 0, mod->init_size); 2371 memset(ptr, 0, mod->init_size);
2205 mod->module_init = ptr; 2372 mod->module_init = ptr;
2206 2373
2207 /* Transfer each section which specifies SHF_ALLOC */ 2374 /* Transfer each section which specifies SHF_ALLOC */
2208 DEBUGP("final section addresses:\n"); 2375 DEBUGP("final section addresses:\n");
2209 for (i = 0; i < hdr->e_shnum; i++) { 2376 for (i = 0; i < info->hdr->e_shnum; i++) {
2210 void *dest; 2377 void *dest;
2378 Elf_Shdr *shdr = &info->sechdrs[i];
2211 2379
2212 if (!(sechdrs[i].sh_flags & SHF_ALLOC)) 2380 if (!(shdr->sh_flags & SHF_ALLOC))
2213 continue; 2381 continue;
2214 2382
2215 if (sechdrs[i].sh_entsize & INIT_OFFSET_MASK) 2383 if (shdr->sh_entsize & INIT_OFFSET_MASK)
2216 dest = mod->module_init 2384 dest = mod->module_init
2217 + (sechdrs[i].sh_entsize & ~INIT_OFFSET_MASK); 2385 + (shdr->sh_entsize & ~INIT_OFFSET_MASK);
2218 else 2386 else
2219 dest = mod->module_core + sechdrs[i].sh_entsize; 2387 dest = mod->module_core + shdr->sh_entsize;
2220 2388
2221 if (sechdrs[i].sh_type != SHT_NOBITS) 2389 if (shdr->sh_type != SHT_NOBITS)
2222 memcpy(dest, (void *)sechdrs[i].sh_addr, 2390 memcpy(dest, (void *)shdr->sh_addr, shdr->sh_size);
2223 sechdrs[i].sh_size);
2224 /* Update sh_addr to point to copy in image. */ 2391 /* Update sh_addr to point to copy in image. */
2225 sechdrs[i].sh_addr = (unsigned long)dest; 2392 shdr->sh_addr = (unsigned long)dest;
2226 DEBUGP("\t0x%lx %s\n", sechdrs[i].sh_addr, secstrings + sechdrs[i].sh_name); 2393 DEBUGP("\t0x%lx %s\n",
2227 } 2394 shdr->sh_addr, info->secstrings + shdr->sh_name);
2228 /* Module has been moved. */
2229 mod = (void *)sechdrs[modindex].sh_addr;
2230 kmemleak_load_module(mod, hdr, sechdrs, secstrings);
2231
2232#if defined(CONFIG_MODULE_UNLOAD)
2233 mod->refptr = alloc_percpu(struct module_ref);
2234 if (!mod->refptr) {
2235 err = -ENOMEM;
2236 goto free_init;
2237 } 2395 }
2238#endif
2239 /* Now we've moved module, initialize linked lists, etc. */
2240 module_unload_init(mod);
2241
2242 /* add kobject, so we can reference it. */
2243 err = mod_sysfs_init(mod);
2244 if (err)
2245 goto free_unload;
2246 2396
2247 /* Set up license info based on the info section */ 2397 return 0;
2248 set_license(mod, get_modinfo(sechdrs, infoindex, "license")); 2398}
2249 2399
2400static int check_module_license_and_versions(struct module *mod)
2401{
2250 /* 2402 /*
2251 * ndiswrapper is under GPL by itself, but loads proprietary modules. 2403 * ndiswrapper is under GPL by itself, but loads proprietary modules.
2252 * Don't use add_taint_module(), as it would prevent ndiswrapper from 2404 * Don't use add_taint_module(), as it would prevent ndiswrapper from
@@ -2259,77 +2411,6 @@ static noinline struct module *load_module(void __user *umod,
2259 if (strcmp(mod->name, "driverloader") == 0) 2411 if (strcmp(mod->name, "driverloader") == 0)
2260 add_taint_module(mod, TAINT_PROPRIETARY_MODULE); 2412 add_taint_module(mod, TAINT_PROPRIETARY_MODULE);
2261 2413
2262 /* Set up MODINFO_ATTR fields */
2263 setup_modinfo(mod, sechdrs, infoindex);
2264
2265 /* Fix up syms, so that st_value is a pointer to location. */
2266 err = simplify_symbols(sechdrs, symindex, strtab, versindex, pcpuindex,
2267 mod);
2268 if (err < 0)
2269 goto cleanup;
2270
2271 /* Now we've got everything in the final locations, we can
2272 * find optional sections. */
2273 mod->kp = section_objs(hdr, sechdrs, secstrings, "__param",
2274 sizeof(*mod->kp), &mod->num_kp);
2275 mod->syms = section_objs(hdr, sechdrs, secstrings, "__ksymtab",
2276 sizeof(*mod->syms), &mod->num_syms);
2277 mod->crcs = section_addr(hdr, sechdrs, secstrings, "__kcrctab");
2278 mod->gpl_syms = section_objs(hdr, sechdrs, secstrings, "__ksymtab_gpl",
2279 sizeof(*mod->gpl_syms),
2280 &mod->num_gpl_syms);
2281 mod->gpl_crcs = section_addr(hdr, sechdrs, secstrings, "__kcrctab_gpl");
2282 mod->gpl_future_syms = section_objs(hdr, sechdrs, secstrings,
2283 "__ksymtab_gpl_future",
2284 sizeof(*mod->gpl_future_syms),
2285 &mod->num_gpl_future_syms);
2286 mod->gpl_future_crcs = section_addr(hdr, sechdrs, secstrings,
2287 "__kcrctab_gpl_future");
2288
2289#ifdef CONFIG_UNUSED_SYMBOLS
2290 mod->unused_syms = section_objs(hdr, sechdrs, secstrings,
2291 "__ksymtab_unused",
2292 sizeof(*mod->unused_syms),
2293 &mod->num_unused_syms);
2294 mod->unused_crcs = section_addr(hdr, sechdrs, secstrings,
2295 "__kcrctab_unused");
2296 mod->unused_gpl_syms = section_objs(hdr, sechdrs, secstrings,
2297 "__ksymtab_unused_gpl",
2298 sizeof(*mod->unused_gpl_syms),
2299 &mod->num_unused_gpl_syms);
2300 mod->unused_gpl_crcs = section_addr(hdr, sechdrs, secstrings,
2301 "__kcrctab_unused_gpl");
2302#endif
2303#ifdef CONFIG_CONSTRUCTORS
2304 mod->ctors = section_objs(hdr, sechdrs, secstrings, ".ctors",
2305 sizeof(*mod->ctors), &mod->num_ctors);
2306#endif
2307
2308#ifdef CONFIG_TRACEPOINTS
2309 mod->tracepoints = section_objs(hdr, sechdrs, secstrings,
2310 "__tracepoints",
2311 sizeof(*mod->tracepoints),
2312 &mod->num_tracepoints);
2313#endif
2314#ifdef CONFIG_EVENT_TRACING
2315 mod->trace_events = section_objs(hdr, sechdrs, secstrings,
2316 "_ftrace_events",
2317 sizeof(*mod->trace_events),
2318 &mod->num_trace_events);
2319 /*
2320 * This section contains pointers to allocated objects in the trace
2321 * code and not scanning it leads to false positives.
2322 */
2323 kmemleak_scan_area(mod->trace_events, sizeof(*mod->trace_events) *
2324 mod->num_trace_events, GFP_KERNEL);
2325#endif
2326#ifdef CONFIG_FTRACE_MCOUNT_RECORD
2327 /* sechdrs[0].sh_size is always zero */
2328 mod->ftrace_callsites = section_objs(hdr, sechdrs, secstrings,
2329 "__mcount_loc",
2330 sizeof(*mod->ftrace_callsites),
2331 &mod->num_ftrace_callsites);
2332#endif
2333#ifdef CONFIG_MODVERSIONS 2414#ifdef CONFIG_MODVERSIONS
2334 if ((mod->num_syms && !mod->crcs) 2415 if ((mod->num_syms && !mod->crcs)
2335 || (mod->num_gpl_syms && !mod->gpl_crcs) 2416 || (mod->num_gpl_syms && !mod->gpl_crcs)
@@ -2339,67 +2420,16 @@ static noinline struct module *load_module(void __user *umod,
2339 || (mod->num_unused_gpl_syms && !mod->unused_gpl_crcs) 2420 || (mod->num_unused_gpl_syms && !mod->unused_gpl_crcs)
2340#endif 2421#endif
2341 ) { 2422 ) {
2342 err = try_to_force_load(mod, 2423 return try_to_force_load(mod,
2343 "no versions for exported symbols"); 2424 "no versions for exported symbols");
2344 if (err)
2345 goto cleanup;
2346 } 2425 }
2347#endif 2426#endif
2427 return 0;
2428}
2348 2429
2349 /* Now do relocations. */ 2430static void flush_module_icache(const struct module *mod)
2350 for (i = 1; i < hdr->e_shnum; i++) { 2431{
2351 const char *strtab = (char *)sechdrs[strindex].sh_addr; 2432 mm_segment_t old_fs;
2352 unsigned int info = sechdrs[i].sh_info;
2353
2354 /* Not a valid relocation section? */
2355 if (info >= hdr->e_shnum)
2356 continue;
2357
2358 /* Don't bother with non-allocated sections */
2359 if (!(sechdrs[info].sh_flags & SHF_ALLOC))
2360 continue;
2361
2362 if (sechdrs[i].sh_type == SHT_REL)
2363 err = apply_relocate(sechdrs, strtab, symindex, i,mod);
2364 else if (sechdrs[i].sh_type == SHT_RELA)
2365 err = apply_relocate_add(sechdrs, strtab, symindex, i,
2366 mod);
2367 if (err < 0)
2368 goto cleanup;
2369 }
2370
2371 /* Find duplicate symbols */
2372 err = verify_export_symbols(mod);
2373 if (err < 0)
2374 goto cleanup;
2375
2376 /* Set up and sort exception table */
2377 mod->extable = section_objs(hdr, sechdrs, secstrings, "__ex_table",
2378 sizeof(*mod->extable), &mod->num_exentries);
2379 sort_extable(mod->extable, mod->extable + mod->num_exentries);
2380
2381 /* Finally, copy percpu area over. */
2382 percpu_modcopy(mod, (void *)sechdrs[pcpuindex].sh_addr,
2383 sechdrs[pcpuindex].sh_size);
2384
2385 add_kallsyms(mod, sechdrs, hdr->e_shnum, symindex, strindex,
2386 symoffs, stroffs, secstrings, strmap);
2387 kfree(strmap);
2388 strmap = NULL;
2389
2390 if (!mod->taints) {
2391 struct _ddebug *debug;
2392 unsigned int num_debug;
2393
2394 debug = section_objs(hdr, sechdrs, secstrings, "__verbose",
2395 sizeof(*debug), &num_debug);
2396 if (debug)
2397 dynamic_debug_setup(debug, num_debug);
2398 }
2399
2400 err = module_finalize(hdr, sechdrs, mod);
2401 if (err < 0)
2402 goto cleanup;
2403 2433
2404 /* flush the icache in correct context */ 2434 /* flush the icache in correct context */
2405 old_fs = get_fs(); 2435 old_fs = get_fs();
@@ -2418,11 +2448,160 @@ static noinline struct module *load_module(void __user *umod,
2418 (unsigned long)mod->module_core + mod->core_size); 2448 (unsigned long)mod->module_core + mod->core_size);
2419 2449
2420 set_fs(old_fs); 2450 set_fs(old_fs);
2451}
2421 2452
2422 mod->args = args; 2453static struct module *layout_and_allocate(struct load_info *info)
2423 if (section_addr(hdr, sechdrs, secstrings, "__obsparm")) 2454{
2424 printk(KERN_WARNING "%s: Ignoring obsolete parameters\n", 2455 /* Module within temporary copy. */
2425 mod->name); 2456 struct module *mod;
2457 Elf_Shdr *pcpusec;
2458 int err;
2459
2460 mod = setup_load_info(info);
2461 if (IS_ERR(mod))
2462 return mod;
2463
2464 err = check_modinfo(mod, info);
2465 if (err)
2466 return ERR_PTR(err);
2467
2468 /* Allow arches to frob section contents and sizes. */
2469 err = module_frob_arch_sections(info->hdr, info->sechdrs,
2470 info->secstrings, mod);
2471 if (err < 0)
2472 goto out;
2473
2474 pcpusec = &info->sechdrs[info->index.pcpu];
2475 if (pcpusec->sh_size) {
2476 /* We have a special allocation for this section. */
2477 err = percpu_modalloc(mod,
2478 pcpusec->sh_size, pcpusec->sh_addralign);
2479 if (err)
2480 goto out;
2481 pcpusec->sh_flags &= ~(unsigned long)SHF_ALLOC;
2482 }
2483
2484 /* Determine total sizes, and put offsets in sh_entsize. For now
2485 this is done generically; there doesn't appear to be any
2486 special cases for the architectures. */
2487 layout_sections(mod, info);
2488
2489 info->strmap = kzalloc(BITS_TO_LONGS(info->sechdrs[info->index.str].sh_size)
2490 * sizeof(long), GFP_KERNEL);
2491 if (!info->strmap) {
2492 err = -ENOMEM;
2493 goto free_percpu;
2494 }
2495 layout_symtab(mod, info);
2496
2497 /* Allocate and move to the final place */
2498 err = move_module(mod, info);
2499 if (err)
2500 goto free_strmap;
2501
2502 /* Module has been copied to its final place now: return it. */
2503 mod = (void *)info->sechdrs[info->index.mod].sh_addr;
2504 kmemleak_load_module(mod, info);
2505 return mod;
2506
2507free_strmap:
2508 kfree(info->strmap);
2509free_percpu:
2510 percpu_modfree(mod);
2511out:
2512 return ERR_PTR(err);
2513}
2514
2515/* mod is no longer valid after this! */
2516static void module_deallocate(struct module *mod, struct load_info *info)
2517{
2518 kfree(info->strmap);
2519 percpu_modfree(mod);
2520 module_free(mod, mod->module_init);
2521 module_free(mod, mod->module_core);
2522}
2523
2524static int post_relocation(struct module *mod, const struct load_info *info)
2525{
2526 /* Sort exception table now relocations are done. */
2527 sort_extable(mod->extable, mod->extable + mod->num_exentries);
2528
2529 /* Copy relocated percpu area over. */
2530 percpu_modcopy(mod, (void *)info->sechdrs[info->index.pcpu].sh_addr,
2531 info->sechdrs[info->index.pcpu].sh_size);
2532
2533 /* Setup kallsyms-specific fields. */
2534 add_kallsyms(mod, info);
2535
2536 /* Arch-specific module finalizing. */
2537 return module_finalize(info->hdr, info->sechdrs, mod);
2538}
2539
2540/* Allocate and load the module: note that size of section 0 is always
2541 zero, and we rely on this for optional sections. */
2542static struct module *load_module(void __user *umod,
2543 unsigned long len,
2544 const char __user *uargs)
2545{
2546 struct load_info info = { NULL, };
2547 struct module *mod;
2548 long err;
2549
2550 DEBUGP("load_module: umod=%p, len=%lu, uargs=%p\n",
2551 umod, len, uargs);
2552
2553 /* Copy in the blobs from userspace, check they are vaguely sane. */
2554 err = copy_and_check(&info, umod, len, uargs);
2555 if (err)
2556 return ERR_PTR(err);
2557
2558 /* Figure out module layout, and allocate all the memory. */
2559 mod = layout_and_allocate(&info);
2560 if (IS_ERR(mod)) {
2561 err = PTR_ERR(mod);
2562 goto free_copy;
2563 }
2564
2565 /* Now module is in final location, initialize linked lists, etc. */
2566 err = module_unload_init(mod);
2567 if (err)
2568 goto free_module;
2569
2570 /* Now we've got everything in the final locations, we can
2571 * find optional sections. */
2572 find_module_sections(mod, &info);
2573
2574 err = check_module_license_and_versions(mod);
2575 if (err)
2576 goto free_unload;
2577
2578 /* Set up MODINFO_ATTR fields */
2579 setup_modinfo(mod, &info);
2580
2581 /* Fix up syms, so that st_value is a pointer to location. */
2582 err = simplify_symbols(mod, &info);
2583 if (err < 0)
2584 goto free_modinfo;
2585
2586 err = apply_relocations(mod, &info);
2587 if (err < 0)
2588 goto free_modinfo;
2589
2590 err = post_relocation(mod, &info);
2591 if (err < 0)
2592 goto free_modinfo;
2593
2594 flush_module_icache(mod);
2595
2596 /* Now copy in args */
2597 mod->args = strndup_user(uargs, ~0UL >> 1);
2598 if (IS_ERR(mod->args)) {
2599 err = PTR_ERR(mod->args);
2600 goto free_arch_cleanup;
2601 }
2602
2603 /* Mark state as coming so strong_try_module_get() ignores us. */
2604 mod->state = MODULE_STATE_COMING;
2426 2605
2427 /* Now sew it into the lists so we can get lockdep and oops 2606 /* Now sew it into the lists so we can get lockdep and oops
2428 * info during argument parsing. Noone should access us, since 2607 * info during argument parsing. Noone should access us, since
@@ -2431,58 +2610,64 @@ static noinline struct module *load_module(void __user *umod,
2431 * function to insert in a way safe to concurrent readers. 2610 * function to insert in a way safe to concurrent readers.
2432 * The mutex protects against concurrent writers. 2611 * The mutex protects against concurrent writers.
2433 */ 2612 */
2613 mutex_lock(&module_mutex);
2614 if (find_module(mod->name)) {
2615 err = -EEXIST;
2616 goto unlock;
2617 }
2618
2619 /* This has to be done once we're sure module name is unique. */
2620 if (!mod->taints)
2621 dynamic_debug_setup(info.debug, info.num_debug);
2622
2623 /* Find duplicate symbols */
2624 err = verify_export_symbols(mod);
2625 if (err < 0)
2626 goto ddebug;
2627
2434 list_add_rcu(&mod->list, &modules); 2628 list_add_rcu(&mod->list, &modules);
2629 mutex_unlock(&module_mutex);
2435 2630
2631 /* Module is ready to execute: parsing args may do that. */
2436 err = parse_args(mod->name, mod->args, mod->kp, mod->num_kp, NULL); 2632 err = parse_args(mod->name, mod->args, mod->kp, mod->num_kp, NULL);
2437 if (err < 0) 2633 if (err < 0)
2438 goto unlink; 2634 goto unlink;
2439 2635
2440 err = mod_sysfs_setup(mod, mod->kp, mod->num_kp); 2636 /* Link in to syfs. */
2637 err = mod_sysfs_setup(mod, &info, mod->kp, mod->num_kp);
2441 if (err < 0) 2638 if (err < 0)
2442 goto unlink; 2639 goto unlink;
2443 add_sect_attrs(mod, hdr->e_shnum, secstrings, sechdrs);
2444 add_notes_attrs(mod, hdr->e_shnum, secstrings, sechdrs);
2445
2446 /* Get rid of temporary copy */
2447 vfree(hdr);
2448 2640
2449 trace_module_load(mod); 2641 /* Get rid of temporary copy and strmap. */
2642 kfree(info.strmap);
2643 free_copy(&info);
2450 2644
2451 /* Done! */ 2645 /* Done! */
2646 trace_module_load(mod);
2452 return mod; 2647 return mod;
2453 2648
2454 unlink: 2649 unlink:
2650 mutex_lock(&module_mutex);
2455 /* Unlink carefully: kallsyms could be walking list. */ 2651 /* Unlink carefully: kallsyms could be walking list. */
2456 list_del_rcu(&mod->list); 2652 list_del_rcu(&mod->list);
2653 ddebug:
2654 if (!mod->taints)
2655 dynamic_debug_remove(info.debug);
2656 unlock:
2657 mutex_unlock(&module_mutex);
2457 synchronize_sched(); 2658 synchronize_sched();
2659 kfree(mod->args);
2660 free_arch_cleanup:
2458 module_arch_cleanup(mod); 2661 module_arch_cleanup(mod);
2459 cleanup: 2662 free_modinfo:
2460 free_modinfo(mod); 2663 free_modinfo(mod);
2461 kobject_del(&mod->mkobj.kobj);
2462 kobject_put(&mod->mkobj.kobj);
2463 free_unload: 2664 free_unload:
2464 module_unload_free(mod); 2665 module_unload_free(mod);
2465#if defined(CONFIG_MODULE_UNLOAD) 2666 free_module:
2466 free_percpu(mod->refptr); 2667 module_deallocate(mod, &info);
2467 free_init: 2668 free_copy:
2468#endif 2669 free_copy(&info);
2469 module_free(mod, mod->module_init);
2470 free_core:
2471 module_free(mod, mod->module_core);
2472 /* mod will be freed with core. Don't access it beyond this line! */
2473 free_percpu:
2474 percpu_modfree(mod);
2475 free_mod:
2476 kfree(args);
2477 kfree(strmap);
2478 free_hdr:
2479 vfree(hdr);
2480 return ERR_PTR(err); 2670 return ERR_PTR(err);
2481
2482 truncated:
2483 printk(KERN_ERR "Module len %lu truncated\n", len);
2484 err = -ENOEXEC;
2485 goto free_hdr;
2486} 2671}
2487 2672
2488/* Call module constructors. */ 2673/* Call module constructors. */
@@ -2507,19 +2692,10 @@ SYSCALL_DEFINE3(init_module, void __user *, umod,
2507 if (!capable(CAP_SYS_MODULE) || modules_disabled) 2692 if (!capable(CAP_SYS_MODULE) || modules_disabled)
2508 return -EPERM; 2693 return -EPERM;
2509 2694
2510 /* Only one module load at a time, please */
2511 if (mutex_lock_interruptible(&module_mutex) != 0)
2512 return -EINTR;
2513
2514 /* Do all the hard work */ 2695 /* Do all the hard work */
2515 mod = load_module(umod, len, uargs); 2696 mod = load_module(umod, len, uargs);
2516 if (IS_ERR(mod)) { 2697 if (IS_ERR(mod))
2517 mutex_unlock(&module_mutex);
2518 return PTR_ERR(mod); 2698 return PTR_ERR(mod);
2519 }
2520
2521 /* Drop lock so they can recurse */
2522 mutex_unlock(&module_mutex);
2523 2699
2524 blocking_notifier_call_chain(&module_notify_list, 2700 blocking_notifier_call_chain(&module_notify_list,
2525 MODULE_STATE_COMING, mod); 2701 MODULE_STATE_COMING, mod);
@@ -2536,9 +2712,7 @@ SYSCALL_DEFINE3(init_module, void __user *, umod,
2536 module_put(mod); 2712 module_put(mod);
2537 blocking_notifier_call_chain(&module_notify_list, 2713 blocking_notifier_call_chain(&module_notify_list,
2538 MODULE_STATE_GOING, mod); 2714 MODULE_STATE_GOING, mod);
2539 mutex_lock(&module_mutex);
2540 free_module(mod); 2715 free_module(mod);
2541 mutex_unlock(&module_mutex);
2542 wake_up(&module_wq); 2716 wake_up(&module_wq);
2543 return ret; 2717 return ret;
2544 } 2718 }
generated by cgit v1.2.2 (git 2.25.0) at 2025-12-16 02:50:06 -0500