aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2015-07-01 13:49:25 -0400
committerLinus Torvalds <torvalds@linux-foundation.org>2015-07-01 13:49:25 -0400
commit02201e3f1b46aed7c6348f406b7b40de80ba6de3 (patch)
tree2392c9098359725c195dd82a72b20ccedc1a1509
parent0890a264794f33df540fbaf274699146903b4e6b (diff)
parent20bdc2cfdbc484777b30b96fcdbb8994038f3ce1 (diff)
Merge tag 'modules-next-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/rusty/linux
Pull module updates from Rusty Russell: "Main excitement here is Peter Zijlstra's lockless rbtree optimization to speed module address lookup. He found some abusers of the module lock doing that too. A little bit of parameter work here too; including Dan Streetman's breaking up the big param mutex so writing a parameter can load another module (yeah, really). Unfortunately that broke the usual suspects, !CONFIG_MODULES and !CONFIG_SYSFS, so those fixes were appended too" * tag 'modules-next-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/rusty/linux: (26 commits) modules: only use mod->param_lock if CONFIG_MODULES param: fix module param locks when !CONFIG_SYSFS. rcu: merge fix for Convert ACCESS_ONCE() to READ_ONCE() and WRITE_ONCE() module: add per-module param_lock module: make perm const params: suppress unused variable error, warn once just in case code changes. modules: clarify CONFIG_MODULE_COMPRESS help, suggest 'N'. kernel/module.c: avoid ifdefs for sig_enforce declaration kernel/workqueue.c: remove ifdefs over wq_power_efficient kernel/params.c: export param_ops_bool_enable_only kernel/params.c: generalize bool_enable_only kernel/module.c: use generic module param operaters for sig_enforce kernel/params: constify struct kernel_param_ops uses sysfs: tightened sysfs permission checks module: Rework module_addr_{min,max} module: Use __module_address() for module_address_lookup() module: Make the mod_tree stuff conditional on PERF_EVENTS || TRACING module: Optimize __module_address() using a latched RB-tree rbtree: Implement generic latch_tree seqlock: Introduce raw_read_seqcount_latch() ...
Diffstat
-rw-r--r--arch/s390/kernel/perf_cpum_sf.c2
-rw-r--r--arch/um/drivers/hostaudio_kern.c20
-rw-r--r--arch/x86/kvm/mmu_audit.c2
-rw-r--r--arch/x86/platform/uv/uv_nmi.c2
-rw-r--r--drivers/block/null_blk.c4
-rw-r--r--drivers/char/ipmi/ipmi_watchdog.c6
-rw-r--r--drivers/dma/dmatest.c4
-rw-r--r--drivers/ide/ide.c2
-rw-r--r--drivers/infiniband/ulp/srp/ib_srp.c4
-rw-r--r--drivers/input/misc/ati_remote2.c4
-rw-r--r--drivers/input/mouse/psmouse-base.c2
-rw-r--r--drivers/misc/lis3lv02d/lis3lv02d.c2
-rw-r--r--drivers/mtd/ubi/block.c2
-rw-r--r--drivers/net/ethernet/myricom/myri10ge/myri10ge.c6
-rw-r--r--drivers/net/wireless/ath/wil6210/main.c4
-rw-r--r--drivers/net/wireless/libertas_tf/if_usb.c6
-rw-r--r--drivers/power/test_power.c16
-rw-r--r--drivers/thermal/intel_powerclamp.c4
-rw-r--r--drivers/tty/hvc/hvc_iucv.c2
-rw-r--r--drivers/tty/sysrq.c2
-rw-r--r--drivers/usb/atm/ueagle-atm.c4
-rw-r--r--drivers/video/fbdev/uvesafb.c2
-rw-r--r--drivers/video/fbdev/vt8623fb.c4
-rw-r--r--drivers/virtio/virtio_mmio.c2
-rw-r--r--fs/nfs/super.c2
-rw-r--r--include/linux/compiler.h15
-rw-r--r--include/linux/kernel.h18
-rw-r--r--include/linux/module.h46
-rw-r--r--include/linux/moduleparam.h99
-rw-r--r--include/linux/rbtree.h16
-rw-r--r--include/linux/rbtree_augmented.h21
-rw-r--r--include/linux/rbtree_latch.h212
-rw-r--r--include/linux/rcupdate.h15
-rw-r--r--include/linux/seqlock.h81
-rw-r--r--init/Kconfig29
-rw-r--r--kernel/jump_label.c10
-rw-r--r--kernel/module.c309
-rw-r--r--kernel/params.c116
-rw-r--r--kernel/time/timekeeping.c29
-rw-r--r--kernel/workqueue.c7
-rw-r--r--lib/bug.c7
-rw-r--r--lib/rbtree.c76
-rw-r--r--net/mac80211/rate.c4
-rw-r--r--net/sunrpc/auth.c2
-rw-r--r--net/sunrpc/xprtsock.c6
-rw-r--r--security/apparmor/lsm.c6
-rw-r--r--security/integrity/ima/ima_crypto.c2
-rw-r--r--sound/pci/hda/hda_intel.c2
48 files changed, 883 insertions, 355 deletions
diff --git a/arch/s390/kernel/perf_cpum_sf.c b/arch/s390/kernel/perf_cpum_sf.c
index e6a1578fc000..afe05bfb7e00 100644
--- a/arch/s390/kernel/perf_cpum_sf.c
+++ b/arch/s390/kernel/perf_cpum_sf.c
@@ -1572,7 +1572,7 @@ static int param_set_sfb_size(const char *val, const struct kernel_param *kp)
1572} 1572}
1573 1573
1574#define param_check_sfb_size(name, p) __param_check(name, p, void) 1574#define param_check_sfb_size(name, p) __param_check(name, p, void)
1575static struct kernel_param_ops param_ops_sfb_size = { 1575static const struct kernel_param_ops param_ops_sfb_size = {
1576 .set = param_set_sfb_size, 1576 .set = param_set_sfb_size,
1577 .get = param_get_sfb_size, 1577 .get = param_get_sfb_size,
1578}; 1578};
diff --git a/arch/um/drivers/hostaudio_kern.c b/arch/um/drivers/hostaudio_kern.c
index 9b90fdc4b151..f6b911cc3923 100644
--- a/arch/um/drivers/hostaudio_kern.c
+++ b/arch/um/drivers/hostaudio_kern.c
@@ -185,9 +185,9 @@ static int hostaudio_open(struct inode *inode, struct file *file)
185 int ret; 185 int ret;
186 186
187#ifdef DEBUG 187#ifdef DEBUG
188 kparam_block_sysfs_write(dsp); 188 kernel_param_lock(THIS_MODULE);
189 printk(KERN_DEBUG "hostaudio: open called (host: %s)\n", dsp); 189 printk(KERN_DEBUG "hostaudio: open called (host: %s)\n", dsp);
190 kparam_unblock_sysfs_write(dsp); 190 kernel_param_unlock(THIS_MODULE);
191#endif 191#endif
192 192
193 state = kmalloc(sizeof(struct hostaudio_state), GFP_KERNEL); 193 state = kmalloc(sizeof(struct hostaudio_state), GFP_KERNEL);
@@ -199,11 +199,11 @@ static int hostaudio_open(struct inode *inode, struct file *file)
199 if (file->f_mode & FMODE_WRITE) 199 if (file->f_mode & FMODE_WRITE)
200 w = 1; 200 w = 1;
201 201
202 kparam_block_sysfs_write(dsp); 202 kernel_param_lock(THIS_MODULE);
203 mutex_lock(&hostaudio_mutex); 203 mutex_lock(&hostaudio_mutex);
204 ret = os_open_file(dsp, of_set_rw(OPENFLAGS(), r, w), 0); 204 ret = os_open_file(dsp, of_set_rw(OPENFLAGS(), r, w), 0);
205 mutex_unlock(&hostaudio_mutex); 205 mutex_unlock(&hostaudio_mutex);
206 kparam_unblock_sysfs_write(dsp); 206 kernel_param_unlock(THIS_MODULE);
207 207
208 if (ret < 0) { 208 if (ret < 0) {
209 kfree(state); 209 kfree(state);
@@ -260,17 +260,17 @@ static int hostmixer_open_mixdev(struct inode *inode, struct file *file)
260 if (file->f_mode & FMODE_WRITE) 260 if (file->f_mode & FMODE_WRITE)
261 w = 1; 261 w = 1;
262 262
263 kparam_block_sysfs_write(mixer); 263 kernel_param_lock(THIS_MODULE);
264 mutex_lock(&hostaudio_mutex); 264 mutex_lock(&hostaudio_mutex);
265 ret = os_open_file(mixer, of_set_rw(OPENFLAGS(), r, w), 0); 265 ret = os_open_file(mixer, of_set_rw(OPENFLAGS(), r, w), 0);
266 mutex_unlock(&hostaudio_mutex); 266 mutex_unlock(&hostaudio_mutex);
267 kparam_unblock_sysfs_write(mixer); 267 kernel_param_unlock(THIS_MODULE);
268 268
269 if (ret < 0) { 269 if (ret < 0) {
270 kparam_block_sysfs_write(dsp); 270 kernel_param_lock(THIS_MODULE);
271 printk(KERN_ERR "hostaudio_open_mixdev failed to open '%s', " 271 printk(KERN_ERR "hostaudio_open_mixdev failed to open '%s', "
272 "err = %d\n", dsp, -ret); 272 "err = %d\n", dsp, -ret);
273 kparam_unblock_sysfs_write(dsp); 273 kernel_param_unlock(THIS_MODULE);
274 kfree(state); 274 kfree(state);
275 return ret; 275 return ret;
276 } 276 }
@@ -326,10 +326,10 @@ MODULE_LICENSE("GPL");
326 326
327static int __init hostaudio_init_module(void) 327static int __init hostaudio_init_module(void)
328{ 328{
329 __kernel_param_lock(); 329 kernel_param_lock(THIS_MODULE);
330 printk(KERN_INFO "UML Audio Relay (host dsp = %s, host mixer = %s)\n", 330 printk(KERN_INFO "UML Audio Relay (host dsp = %s, host mixer = %s)\n",
331 dsp, mixer); 331 dsp, mixer);
332 __kernel_param_unlock(); 332 kernel_param_unlock(THIS_MODULE);
333 333
334 module_data.dev_audio = register_sound_dsp(&hostaudio_fops, -1); 334 module_data.dev_audio = register_sound_dsp(&hostaudio_fops, -1);
335 if (module_data.dev_audio < 0) { 335 if (module_data.dev_audio < 0) {
diff --git a/arch/x86/kvm/mmu_audit.c b/arch/x86/kvm/mmu_audit.c
index a4f62e6f2db2..03d518e499a6 100644
--- a/arch/x86/kvm/mmu_audit.c
+++ b/arch/x86/kvm/mmu_audit.c
@@ -297,7 +297,7 @@ static int mmu_audit_set(const char *val, const struct kernel_param *kp)
297 return 0; 297 return 0;
298} 298}
299 299
300static struct kernel_param_ops audit_param_ops = { 300static const struct kernel_param_ops audit_param_ops = {
301 .set = mmu_audit_set, 301 .set = mmu_audit_set,
302 .get = param_get_bool, 302 .get = param_get_bool,
303}; 303};
diff --git a/arch/x86/platform/uv/uv_nmi.c b/arch/x86/platform/uv/uv_nmi.c
index 7488cafab955..020c101c255f 100644
--- a/arch/x86/platform/uv/uv_nmi.c
+++ b/arch/x86/platform/uv/uv_nmi.c
@@ -104,7 +104,7 @@ static int param_set_local64(const char *val, const struct kernel_param *kp)
104 return 0; 104 return 0;
105} 105}
106 106
107static struct kernel_param_ops param_ops_local64 = { 107static const struct kernel_param_ops param_ops_local64 = {
108 .get = param_get_local64, 108 .get = param_get_local64,
109 .set = param_set_local64, 109 .set = param_set_local64,
110}; 110};
diff --git a/drivers/block/null_blk.c b/drivers/block/null_blk.c
index 6f9b7534928e..69de41a87b74 100644
--- a/drivers/block/null_blk.c
+++ b/drivers/block/null_blk.c
@@ -99,7 +99,7 @@ static int null_set_queue_mode(const char *str, const struct kernel_param *kp)
99 return null_param_store_val(str, &queue_mode, NULL_Q_BIO, NULL_Q_MQ); 99 return null_param_store_val(str, &queue_mode, NULL_Q_BIO, NULL_Q_MQ);
100} 100}
101 101
102static struct kernel_param_ops null_queue_mode_param_ops = { 102static const struct kernel_param_ops null_queue_mode_param_ops = {
103 .set = null_set_queue_mode, 103 .set = null_set_queue_mode,
104 .get = param_get_int, 104 .get = param_get_int,
105}; 105};
@@ -127,7 +127,7 @@ static int null_set_irqmode(const char *str, const struct kernel_param *kp)
127 NULL_IRQ_TIMER); 127 NULL_IRQ_TIMER);
128} 128}
129 129
130static struct kernel_param_ops null_irqmode_param_ops = { 130static const struct kernel_param_ops null_irqmode_param_ops = {
131 .set = null_set_irqmode, 131 .set = null_set_irqmode,
132 .get = param_get_int, 132 .get = param_get_int,
133}; 133};
diff --git a/drivers/char/ipmi/ipmi_watchdog.c b/drivers/char/ipmi/ipmi_watchdog.c
index 37b8be7cba95..0ac3bd1a5497 100644
--- a/drivers/char/ipmi/ipmi_watchdog.c
+++ b/drivers/char/ipmi/ipmi_watchdog.c
@@ -208,7 +208,7 @@ static int set_param_timeout(const char *val, const struct kernel_param *kp)
208 return rv; 208 return rv;
209} 209}
210 210
211static struct kernel_param_ops param_ops_timeout = { 211static const struct kernel_param_ops param_ops_timeout = {
212 .set = set_param_timeout, 212 .set = set_param_timeout,
213 .get = param_get_int, 213 .get = param_get_int,
214}; 214};
@@ -270,14 +270,14 @@ static int set_param_wdog_ifnum(const char *val, const struct kernel_param *kp)
270 return 0; 270 return 0;
271} 271}
272 272
273static struct kernel_param_ops param_ops_wdog_ifnum = { 273static const struct kernel_param_ops param_ops_wdog_ifnum = {
274 .set = set_param_wdog_ifnum, 274 .set = set_param_wdog_ifnum,
275 .get = param_get_int, 275 .get = param_get_int,
276}; 276};
277 277
278#define param_check_wdog_ifnum param_check_int 278#define param_check_wdog_ifnum param_check_int
279 279
280static struct kernel_param_ops param_ops_str = { 280static const struct kernel_param_ops param_ops_str = {
281 .set = set_param_str, 281 .set = set_param_str,
282 .get = get_param_str, 282 .get = get_param_str,
283}; 283};
diff --git a/drivers/dma/dmatest.c b/drivers/dma/dmatest.c
index 220ee49633e4..b8576fd6bd0e 100644
--- a/drivers/dma/dmatest.c
+++ b/drivers/dma/dmatest.c
@@ -120,7 +120,7 @@ static struct dmatest_info {
120 120
121static int dmatest_run_set(const char *val, const struct kernel_param *kp); 121static int dmatest_run_set(const char *val, const struct kernel_param *kp);
122static int dmatest_run_get(char *val, const struct kernel_param *kp); 122static int dmatest_run_get(char *val, const struct kernel_param *kp);
123static struct kernel_param_ops run_ops = { 123static const struct kernel_param_ops run_ops = {
124 .set = dmatest_run_set, 124 .set = dmatest_run_set,
125 .get = dmatest_run_get, 125 .get = dmatest_run_get,
126}; 126};
@@ -195,7 +195,7 @@ static int dmatest_wait_get(char *val, const struct kernel_param *kp)
195 return param_get_bool(val, kp); 195 return param_get_bool(val, kp);
196} 196}
197 197
198static struct kernel_param_ops wait_ops = { 198static const struct kernel_param_ops wait_ops = {
199 .get = dmatest_wait_get, 199 .get = dmatest_wait_get,
200 .set = param_set_bool, 200 .set = param_set_bool,
201}; 201};
diff --git a/drivers/ide/ide.c b/drivers/ide/ide.c
index e29b02ca9e91..f086ef387475 100644
--- a/drivers/ide/ide.c
+++ b/drivers/ide/ide.c
@@ -199,7 +199,7 @@ static int ide_set_dev_param_mask(const char *s, const struct kernel_param *kp)
199 return 0; 199 return 0;
200} 200}
201 201
202static struct kernel_param_ops param_ops_ide_dev_mask = { 202static const struct kernel_param_ops param_ops_ide_dev_mask = {
203 .set = ide_set_dev_param_mask 203 .set = ide_set_dev_param_mask
204}; 204};
205 205
diff --git a/drivers/infiniband/ulp/srp/ib_srp.c b/drivers/infiniband/ulp/srp/ib_srp.c
index eada8f758ad4..267dc4f75502 100644
--- a/drivers/infiniband/ulp/srp/ib_srp.c
+++ b/drivers/infiniband/ulp/srp/ib_srp.c
@@ -99,7 +99,7 @@ module_param(register_always, bool, 0444);
99MODULE_PARM_DESC(register_always, 99MODULE_PARM_DESC(register_always,
100 "Use memory registration even for contiguous memory regions"); 100 "Use memory registration even for contiguous memory regions");
101 101
102static struct kernel_param_ops srp_tmo_ops; 102static const struct kernel_param_ops srp_tmo_ops;
103 103
104static int srp_reconnect_delay = 10; 104static int srp_reconnect_delay = 10;
105module_param_cb(reconnect_delay, &srp_tmo_ops, &srp_reconnect_delay, 105module_param_cb(reconnect_delay, &srp_tmo_ops, &srp_reconnect_delay,
@@ -184,7 +184,7 @@ out:
184 return res; 184 return res;
185} 185}
186 186
187static struct kernel_param_ops srp_tmo_ops = { 187static const struct kernel_param_ops srp_tmo_ops = {
188 .get = srp_tmo_get, 188 .get = srp_tmo_get,
189 .set = srp_tmo_set, 189 .set = srp_tmo_set,
190}; 190};
diff --git a/drivers/input/misc/ati_remote2.c b/drivers/input/misc/ati_remote2.c
index f63341f20b91..cfd58e87da26 100644
--- a/drivers/input/misc/ati_remote2.c
+++ b/drivers/input/misc/ati_remote2.c
@@ -94,7 +94,7 @@ static int ati_remote2_get_mode_mask(char *buffer,
94 94
95static unsigned int channel_mask = ATI_REMOTE2_MAX_CHANNEL_MASK; 95static unsigned int channel_mask = ATI_REMOTE2_MAX_CHANNEL_MASK;
96#define param_check_channel_mask(name, p) __param_check(name, p, unsigned int) 96#define param_check_channel_mask(name, p) __param_check(name, p, unsigned int)
97static struct kernel_param_ops param_ops_channel_mask = { 97static const struct kernel_param_ops param_ops_channel_mask = {
98 .set = ati_remote2_set_channel_mask, 98 .set = ati_remote2_set_channel_mask,
99 .get = ati_remote2_get_channel_mask, 99 .get = ati_remote2_get_channel_mask,
100}; 100};
@@ -103,7 +103,7 @@ MODULE_PARM_DESC(channel_mask, "Bitmask of channels to accept <15:Channel16>...<
103 103
104static unsigned int mode_mask = ATI_REMOTE2_MAX_MODE_MASK; 104static unsigned int mode_mask = ATI_REMOTE2_MAX_MODE_MASK;
105#define param_check_mode_mask(name, p) __param_check(name, p, unsigned int) 105#define param_check_mode_mask(name, p) __param_check(name, p, unsigned int)
106static struct kernel_param_ops param_ops_mode_mask = { 106static const struct kernel_param_ops param_ops_mode_mask = {
107 .set = ati_remote2_set_mode_mask, 107 .set = ati_remote2_set_mode_mask,
108 .get = ati_remote2_get_mode_mask, 108 .get = ati_remote2_get_mode_mask,
109}; 109};
diff --git a/drivers/input/mouse/psmouse-base.c b/drivers/input/mouse/psmouse-base.c
index 7c4ba43d253e..ec3477036150 100644
--- a/drivers/input/mouse/psmouse-base.c
+++ b/drivers/input/mouse/psmouse-base.c
@@ -47,7 +47,7 @@ MODULE_LICENSE("GPL");
47static unsigned int psmouse_max_proto = PSMOUSE_AUTO; 47static unsigned int psmouse_max_proto = PSMOUSE_AUTO;
48static int psmouse_set_maxproto(const char *val, const struct kernel_param *); 48static int psmouse_set_maxproto(const char *val, const struct kernel_param *);
49static int psmouse_get_maxproto(char *buffer, const struct kernel_param *kp); 49static int psmouse_get_maxproto(char *buffer, const struct kernel_param *kp);
50static struct kernel_param_ops param_ops_proto_abbrev = { 50static const struct kernel_param_ops param_ops_proto_abbrev = {
51 .set = psmouse_set_maxproto, 51 .set = psmouse_set_maxproto,
52 .get = psmouse_get_maxproto, 52 .get = psmouse_get_maxproto,
53}; 53};
diff --git a/drivers/misc/lis3lv02d/lis3lv02d.c b/drivers/misc/lis3lv02d/lis3lv02d.c
index 4739689d23ad..fb8705fc3aca 100644
--- a/drivers/misc/lis3lv02d/lis3lv02d.c
+++ b/drivers/misc/lis3lv02d/lis3lv02d.c
@@ -115,7 +115,7 @@ static int param_set_axis(const char *val, const struct kernel_param *kp)
115 return ret; 115 return ret;
116} 116}
117 117
118static struct kernel_param_ops param_ops_axis = { 118static const struct kernel_param_ops param_ops_axis = {
119 .set = param_set_axis, 119 .set = param_set_axis,
120 .get = param_get_int, 120 .get = param_get_int,
121}; 121};
diff --git a/drivers/mtd/ubi/block.c b/drivers/mtd/ubi/block.c
index 1a92d30689e7..ebf46ad2d513 100644
--- a/drivers/mtd/ubi/block.c
+++ b/drivers/mtd/ubi/block.c
@@ -162,7 +162,7 @@ static int __init ubiblock_set_param(const char *val,
162 return 0; 162 return 0;
163} 163}
164 164
165static struct kernel_param_ops ubiblock_param_ops = { 165static const struct kernel_param_ops ubiblock_param_ops = {
166 .set = ubiblock_set_param, 166 .set = ubiblock_set_param,
167}; 167};
168module_param_cb(block, &ubiblock_param_ops, NULL, 0); 168module_param_cb(block, &ubiblock_param_ops, NULL, 0);
diff --git a/drivers/net/ethernet/myricom/myri10ge/myri10ge.c b/drivers/net/ethernet/myricom/myri10ge/myri10ge.c
index 2bae50292dcd..83651ac8ddb9 100644
--- a/drivers/net/ethernet/myricom/myri10ge/myri10ge.c
+++ b/drivers/net/ethernet/myricom/myri10ge/myri10ge.c
@@ -279,7 +279,7 @@ MODULE_FIRMWARE("myri10ge_eth_z8e.dat");
279MODULE_FIRMWARE("myri10ge_rss_ethp_z8e.dat"); 279MODULE_FIRMWARE("myri10ge_rss_ethp_z8e.dat");
280MODULE_FIRMWARE("myri10ge_rss_eth_z8e.dat"); 280MODULE_FIRMWARE("myri10ge_rss_eth_z8e.dat");
281 281
282/* Careful: must be accessed under kparam_block_sysfs_write */ 282/* Careful: must be accessed under kernel_param_lock() */
283static char *myri10ge_fw_name = NULL; 283static char *myri10ge_fw_name = NULL;
284module_param(myri10ge_fw_name, charp, S_IRUGO | S_IWUSR); 284module_param(myri10ge_fw_name, charp, S_IRUGO | S_IWUSR);
285MODULE_PARM_DESC(myri10ge_fw_name, "Firmware image name"); 285MODULE_PARM_DESC(myri10ge_fw_name, "Firmware image name");
@@ -3427,7 +3427,7 @@ static void myri10ge_select_firmware(struct myri10ge_priv *mgp)
3427 } 3427 }
3428 } 3428 }
3429 3429
3430 kparam_block_sysfs_write(myri10ge_fw_name); 3430 kernel_param_lock(THIS_MODULE);
3431 if (myri10ge_fw_name != NULL) { 3431 if (myri10ge_fw_name != NULL) {
3432 char *fw_name = kstrdup(myri10ge_fw_name, GFP_KERNEL); 3432 char *fw_name = kstrdup(myri10ge_fw_name, GFP_KERNEL);
3433 if (fw_name) { 3433 if (fw_name) {
@@ -3435,7 +3435,7 @@ static void myri10ge_select_firmware(struct myri10ge_priv *mgp)
3435 set_fw_name(mgp, fw_name, true); 3435 set_fw_name(mgp, fw_name, true);
3436 } 3436 }
3437 } 3437 }
3438 kparam_unblock_sysfs_write(myri10ge_fw_name); 3438 kernel_param_unlock(THIS_MODULE);
3439 3439
3440 if (mgp->board_number < MYRI10GE_MAX_BOARDS && 3440 if (mgp->board_number < MYRI10GE_MAX_BOARDS &&
3441 myri10ge_fw_names[mgp->board_number] != NULL && 3441 myri10ge_fw_names[mgp->board_number] != NULL &&
diff --git a/drivers/net/wireless/ath/wil6210/main.c b/drivers/net/wireless/ath/wil6210/main.c
index b9febab89167..6ca6193ab8a6 100644
--- a/drivers/net/wireless/ath/wil6210/main.c
+++ b/drivers/net/wireless/ath/wil6210/main.c
@@ -62,7 +62,7 @@ static int mtu_max_set(const char *val, const struct kernel_param *kp)
62 return ret; 62 return ret;
63} 63}
64 64
65static struct kernel_param_ops mtu_max_ops = { 65static const struct kernel_param_ops mtu_max_ops = {
66 .set = mtu_max_set, 66 .set = mtu_max_set,
67 .get = param_get_uint, 67 .get = param_get_uint,
68}; 68};
@@ -91,7 +91,7 @@ static int ring_order_set(const char *val, const struct kernel_param *kp)
91 return 0; 91 return 0;
92} 92}
93 93
94static struct kernel_param_ops ring_order_ops = { 94static const struct kernel_param_ops ring_order_ops = {
95 .set = ring_order_set, 95 .set = ring_order_set,
96 .get = param_get_uint, 96 .get = param_get_uint,
97}; 97};
diff --git a/drivers/net/wireless/libertas_tf/if_usb.c b/drivers/net/wireless/libertas_tf/if_usb.c
index 1a20cee5febe..799a2efe5793 100644
--- a/drivers/net/wireless/libertas_tf/if_usb.c
+++ b/drivers/net/wireless/libertas_tf/if_usb.c
@@ -821,15 +821,15 @@ static int if_usb_prog_firmware(struct if_usb_card *cardp)
821 821
822 lbtf_deb_enter(LBTF_DEB_USB); 822 lbtf_deb_enter(LBTF_DEB_USB);
823 823
824 kparam_block_sysfs_write(fw_name); 824 kernel_param_lock(THIS_MODULE);
825 ret = request_firmware(&cardp->fw, lbtf_fw_name, &cardp->udev->dev); 825 ret = request_firmware(&cardp->fw, lbtf_fw_name, &cardp->udev->dev);
826 if (ret < 0) { 826 if (ret < 0) {
827 pr_err("request_firmware() failed with %#x\n", ret); 827 pr_err("request_firmware() failed with %#x\n", ret);
828 pr_err("firmware %s not found\n", lbtf_fw_name); 828 pr_err("firmware %s not found\n", lbtf_fw_name);
829 kparam_unblock_sysfs_write(fw_name); 829 kernel_param_unlock(THIS_MODULE);
830 goto done; 830 goto done;
831 } 831 }
832 kparam_unblock_sysfs_write(fw_name); 832 kernel_param_unlock(THIS_MODULE);
833 833
834 if (check_fwfile_format(cardp->fw->data, cardp->fw->size)) 834 if (check_fwfile_format(cardp->fw->data, cardp->fw->size))
835 goto release_fw; 835 goto release_fw;
diff --git a/drivers/power/test_power.c b/drivers/power/test_power.c
index f986e0cca7ac..83c42ea88f2b 100644
--- a/drivers/power/test_power.c
+++ b/drivers/power/test_power.c
@@ -448,42 +448,42 @@ static int param_set_battery_voltage(const char *key,
448 448
449#define param_get_battery_voltage param_get_int 449#define param_get_battery_voltage param_get_int
450 450
451static struct kernel_param_ops param_ops_ac_online = { 451static const struct kernel_param_ops param_ops_ac_online = {
452 .set = param_set_ac_online, 452 .set = param_set_ac_online,
453 .get = param_get_ac_online, 453 .get = param_get_ac_online,
454}; 454};
455 455
456static struct kernel_param_ops param_ops_usb_online = { 456static const struct kernel_param_ops param_ops_usb_online = {
457 .set = param_set_usb_online, 457 .set = param_set_usb_online,
458 .get = param_get_usb_online, 458 .get = param_get_usb_online,
459}; 459};
460 460
461static struct kernel_param_ops param_ops_battery_status = { 461static const struct kernel_param_ops param_ops_battery_status = {
462 .set = param_set_battery_status, 462 .set = param_set_battery_status,
463 .get = param_get_battery_status, 463 .get = param_get_battery_status,
464}; 464};
465 465
466static struct kernel_param_ops param_ops_battery_present = { 466static const struct kernel_param_ops param_ops_battery_present = {
467 .set = param_set_battery_present, 467 .set = param_set_battery_present,
468 .get = param_get_battery_present, 468 .get = param_get_battery_present,
469}; 469};
470 470
471static struct kernel_param_ops param_ops_battery_technology = { 471static const struct kernel_param_ops param_ops_battery_technology = {
472 .set = param_set_battery_technology, 472 .set = param_set_battery_technology,
473 .get = param_get_battery_technology, 473 .get = param_get_battery_technology,
474}; 474};
475 475
476static struct kernel_param_ops param_ops_battery_health = { 476static const struct kernel_param_ops param_ops_battery_health = {
477 .set = param_set_battery_health, 477 .set = param_set_battery_health,
478 .get = param_get_battery_health, 478 .get = param_get_battery_health,
479}; 479};
480 480
481static struct kernel_param_ops param_ops_battery_capacity = { 481static const struct kernel_param_ops param_ops_battery_capacity = {
482 .set = param_set_battery_capacity, 482 .set = param_set_battery_capacity,
483 .get = param_get_battery_capacity, 483 .get = param_get_battery_capacity,
484}; 484};
485 485
486static struct kernel_param_ops param_ops_battery_voltage = { 486static const struct kernel_param_ops param_ops_battery_voltage = {
487 .set = param_set_battery_voltage, 487 .set = param_set_battery_voltage,
488 .get = param_get_battery_voltage, 488 .get = param_get_battery_voltage,
489}; 489};
diff --git a/drivers/thermal/intel_powerclamp.c b/drivers/thermal/intel_powerclamp.c
index 2e6716104d3f..5820e8513927 100644
--- a/drivers/thermal/intel_powerclamp.c
+++ b/drivers/thermal/intel_powerclamp.c
@@ -119,7 +119,7 @@ exit:
119 return ret; 119 return ret;
120} 120}
121 121
122static struct kernel_param_ops duration_ops = { 122static const struct kernel_param_ops duration_ops = {
123 .set = duration_set, 123 .set = duration_set,
124 .get = param_get_int, 124 .get = param_get_int,
125}; 125};
@@ -167,7 +167,7 @@ exit_win:
167 return ret; 167 return ret;
168} 168}
169 169
170static struct kernel_param_ops window_size_ops = { 170static const struct kernel_param_ops window_size_ops = {
171 .set = window_size_set, 171 .set = window_size_set,
172 .get = param_get_int, 172 .get = param_get_int,
173}; 173};
diff --git a/drivers/tty/hvc/hvc_iucv.c b/drivers/tty/hvc/hvc_iucv.c
index f78a87b07872..bb809cf36617 100644
--- a/drivers/tty/hvc/hvc_iucv.c
+++ b/drivers/tty/hvc/hvc_iucv.c
@@ -1345,7 +1345,7 @@ static int param_get_vmidfilter(char *buffer, const struct kernel_param *kp)
1345 1345
1346#define param_check_vmidfilter(name, p) __param_check(name, p, void) 1346#define param_check_vmidfilter(name, p) __param_check(name, p, void)
1347 1347
1348static struct kernel_param_ops param_ops_vmidfilter = { 1348static const struct kernel_param_ops param_ops_vmidfilter = {
1349 .set = param_set_vmidfilter, 1349 .set = param_set_vmidfilter,
1350 .get = param_get_vmidfilter, 1350 .get = param_get_vmidfilter,
1351}; 1351};
diff --git a/drivers/tty/sysrq.c b/drivers/tty/sysrq.c
index 2847108cc8dd..b5b427888b24 100644
--- a/drivers/tty/sysrq.c
+++ b/drivers/tty/sysrq.c
@@ -988,7 +988,7 @@ static int sysrq_reset_seq_param_set(const char *buffer,
988 return 0; 988 return 0;
989} 989}
990 990
991static struct kernel_param_ops param_ops_sysrq_reset_seq = { 991static const struct kernel_param_ops param_ops_sysrq_reset_seq = {
992 .get = param_get_ushort, 992 .get = param_get_ushort,
993 .set = sysrq_reset_seq_param_set, 993 .set = sysrq_reset_seq_param_set,
994}; 994};
diff --git a/drivers/usb/atm/ueagle-atm.c b/drivers/usb/atm/ueagle-atm.c
index 888998a7fe31..a2ae88dbda78 100644
--- a/drivers/usb/atm/ueagle-atm.c
+++ b/drivers/usb/atm/ueagle-atm.c
@@ -1599,7 +1599,7 @@ static void cmvs_file_name(struct uea_softc *sc, char *const cmv_name, int ver)
1599 char file_arr[] = "CMVxy.bin"; 1599 char file_arr[] = "CMVxy.bin";
1600 char *file; 1600 char *file;
1601 1601
1602 kparam_block_sysfs_write(cmv_file); 1602 kernel_param_lock(THIS_MODULE);
1603 /* set proper name corresponding modem version and line type */ 1603 /* set proper name corresponding modem version and line type */
1604 if (cmv_file[sc->modem_index] == NULL) { 1604 if (cmv_file[sc->modem_index] == NULL) {
1605 if (UEA_CHIP_VERSION(sc) == ADI930) 1605 if (UEA_CHIP_VERSION(sc) == ADI930)
@@ -1618,7 +1618,7 @@ static void cmvs_file_name(struct uea_softc *sc, char *const cmv_name, int ver)
1618 strlcat(cmv_name, file, UEA_FW_NAME_MAX); 1618 strlcat(cmv_name, file, UEA_FW_NAME_MAX);
1619 if (ver == 2) 1619 if (ver == 2)
1620 strlcat(cmv_name, ".v2", UEA_FW_NAME_MAX); 1620 strlcat(cmv_name, ".v2", UEA_FW_NAME_MAX);
1621 kparam_unblock_sysfs_write(cmv_file); 1621 kernel_param_unlock(THIS_MODULE);
1622} 1622}
1623 1623
1624static int request_cmvs_old(struct uea_softc *sc, 1624static int request_cmvs_old(struct uea_softc *sc,
diff --git a/drivers/video/fbdev/uvesafb.c b/drivers/video/fbdev/uvesafb.c
index d32d1c4d1b99..178ae93b7ebd 100644
--- a/drivers/video/fbdev/uvesafb.c
+++ b/drivers/video/fbdev/uvesafb.c
@@ -1977,7 +1977,7 @@ static int param_set_scroll(const char *val, const struct kernel_param *kp)
1977 1977
1978 return 0; 1978 return 0;
1979} 1979}
1980static struct kernel_param_ops param_ops_scroll = { 1980static const struct kernel_param_ops param_ops_scroll = {
1981 .set = param_set_scroll, 1981 .set = param_set_scroll,
1982}; 1982};
1983#define param_check_scroll(name, p) __param_check(name, p, void) 1983#define param_check_scroll(name, p) __param_check(name, p, void)
diff --git a/drivers/video/fbdev/vt8623fb.c b/drivers/video/fbdev/vt8623fb.c
index ea7f056ed5fe..8bac309c24b9 100644
--- a/drivers/video/fbdev/vt8623fb.c
+++ b/drivers/video/fbdev/vt8623fb.c
@@ -754,9 +754,9 @@ static int vt8623_pci_probe(struct pci_dev *dev, const struct pci_device_id *id)
754 754
755 /* Prepare startup mode */ 755 /* Prepare startup mode */
756 756
757 kparam_block_sysfs_write(mode_option); 757 kernel_param_lock(THIS_MODULE);
758 rc = fb_find_mode(&(info->var), info, mode_option, NULL, 0, NULL, 8); 758 rc = fb_find_mode(&(info->var), info, mode_option, NULL, 0, NULL, 8);
759 kparam_unblock_sysfs_write(mode_option); 759 kernel_param_unlock(THIS_MODULE);
760 if (! ((rc == 1) || (rc == 2))) { 760 if (! ((rc == 1) || (rc == 2))) {
761 rc = -EINVAL; 761 rc = -EINVAL;
762 dev_err(info->device, "mode %s not found\n", mode_option); 762 dev_err(info->device, "mode %s not found\n", mode_option);
diff --git a/drivers/virtio/virtio_mmio.c b/drivers/virtio/virtio_mmio.c
index 7a5e60dea6c5..10189b5b627f 100644
--- a/drivers/virtio/virtio_mmio.c
+++ b/drivers/virtio/virtio_mmio.c
@@ -691,7 +691,7 @@ static int vm_cmdline_get(char *buffer, const struct kernel_param *kp)
691 return strlen(buffer) + 1; 691 return strlen(buffer) + 1;
692} 692}
693 693
694static struct kernel_param_ops vm_cmdline_param_ops = { 694static const struct kernel_param_ops vm_cmdline_param_ops = {
695 .set = vm_cmdline_set, 695 .set = vm_cmdline_set,
696 .get = vm_cmdline_get, 696 .get = vm_cmdline_get,
697}; 697};
diff --git a/fs/nfs/super.c b/fs/nfs/super.c
index f175b833b6ba..aa62004f1706 100644
--- a/fs/nfs/super.c
+++ b/fs/nfs/super.c
@@ -2847,7 +2847,7 @@ static int param_set_portnr(const char *val, const struct kernel_param *kp)
2847 *((unsigned int *)kp->arg) = num; 2847 *((unsigned int *)kp->arg) = num;
2848 return 0; 2848 return 0;
2849} 2849}
2850static struct kernel_param_ops param_ops_portnr = { 2850static const struct kernel_param_ops param_ops_portnr = {
2851 .set = param_set_portnr, 2851 .set = param_set_portnr,
2852 .get = param_get_uint, 2852 .get = param_get_uint,
2853}; 2853};
diff --git a/include/linux/compiler.h b/include/linux/compiler.h
index 26fc8bc77f85..7f8ad9593da7 100644
--- a/include/linux/compiler.h
+++ b/include/linux/compiler.h
@@ -475,6 +475,21 @@ static __always_inline void __write_once_size(volatile void *p, void *res, int s
475 (volatile typeof(x) *)&(x); }) 475 (volatile typeof(x) *)&(x); })
476#define ACCESS_ONCE(x) (*__ACCESS_ONCE(x)) 476#define ACCESS_ONCE(x) (*__ACCESS_ONCE(x))
477 477
478/**
479 * lockless_dereference() - safely load a pointer for later dereference
480 * @p: The pointer to load
481 *
482 * Similar to rcu_dereference(), but for situations where the pointed-to
483 * object's lifetime is managed by something other than RCU. That
484 * "something other" might be reference counting or simple immortality.
485 */
486#define lockless_dereference(p) \
487({ \
488 typeof(p) _________p1 = READ_ONCE(p); \
489 smp_read_barrier_depends(); /* Dependency order vs. p above. */ \
490 (_________p1); \
491})
492
478/* Ignore/forbid kprobes attach on very low level functions marked by this attribute: */ 493/* Ignore/forbid kprobes attach on very low level functions marked by this attribute: */
479#ifdef CONFIG_KPROBES 494#ifdef CONFIG_KPROBES
480# define __kprobes __attribute__((__section__(".kprobes.text"))) 495# define __kprobes __attribute__((__section__(".kprobes.text")))
diff --git a/include/linux/kernel.h b/include/linux/kernel.h
index 5acf5b70866d..cfa9351c7536 100644
--- a/include/linux/kernel.h
+++ b/include/linux/kernel.h
@@ -813,13 +813,15 @@ static inline void ftrace_dump(enum ftrace_dump_mode oops_dump_mode) { }
813#endif 813#endif
814 814
815/* Permissions on a sysfs file: you didn't miss the 0 prefix did you? */ 815/* Permissions on a sysfs file: you didn't miss the 0 prefix did you? */
816#define VERIFY_OCTAL_PERMISSIONS(perms) \ 816#define VERIFY_OCTAL_PERMISSIONS(perms) \
817 (BUILD_BUG_ON_ZERO((perms) < 0) + \ 817 (BUILD_BUG_ON_ZERO((perms) < 0) + \
818 BUILD_BUG_ON_ZERO((perms) > 0777) + \ 818 BUILD_BUG_ON_ZERO((perms) > 0777) + \
819 /* User perms >= group perms >= other perms */ \ 819 /* USER_READABLE >= GROUP_READABLE >= OTHER_READABLE */ \
820 BUILD_BUG_ON_ZERO(((perms) >> 6) < (((perms) >> 3) & 7)) + \ 820 BUILD_BUG_ON_ZERO((((perms) >> 6) & 4) < (((perms) >> 3) & 4)) + \
821 BUILD_BUG_ON_ZERO((((perms) >> 3) & 7) < ((perms) & 7)) + \ 821 BUILD_BUG_ON_ZERO((((perms) >> 3) & 4) < ((perms) & 4)) + \
822 /* Other writable? Generally considered a bad idea. */ \ 822 /* USER_WRITABLE >= GROUP_WRITABLE */ \
823 BUILD_BUG_ON_ZERO((perms) & 2) + \ 823 BUILD_BUG_ON_ZERO((((perms) >> 6) & 2) < (((perms) >> 3) & 2)) + \
824 /* OTHER_WRITABLE? Generally considered a bad idea. */ \
825 BUILD_BUG_ON_ZERO((perms) & 2) + \
824 (perms)) 826 (perms))
825#endif 827#endif
diff --git a/include/linux/module.h b/include/linux/module.h
index 7ffe0851d244..d67b1932cc59 100644
--- a/include/linux/module.h
+++ b/include/linux/module.h
@@ -17,6 +17,7 @@
17#include <linux/moduleparam.h> 17#include <linux/moduleparam.h>
18#include <linux/jump_label.h> 18#include <linux/jump_label.h>
19#include <linux/export.h> 19#include <linux/export.h>
20#include <linux/rbtree_latch.h>
20 21
21#include <linux/percpu.h> 22#include <linux/percpu.h>
22#include <asm/module.h> 23#include <asm/module.h>
@@ -210,6 +211,13 @@ enum module_state {
210 MODULE_STATE_UNFORMED, /* Still setting it up. */ 211 MODULE_STATE_UNFORMED, /* Still setting it up. */
211}; 212};
212 213
214struct module;
215
216struct mod_tree_node {
217 struct module *mod;
218 struct latch_tree_node node;
219};
220
213struct module { 221struct module {
214 enum module_state state; 222 enum module_state state;
215 223
@@ -232,6 +240,9 @@ struct module {
232 unsigned int num_syms; 240 unsigned int num_syms;
233 241
234 /* Kernel parameters. */ 242 /* Kernel parameters. */
243#ifdef CONFIG_SYSFS
244 struct mutex param_lock;
245#endif
235 struct kernel_param *kp; 246 struct kernel_param *kp;
236 unsigned int num_kp; 247 unsigned int num_kp;
237 248
@@ -271,8 +282,15 @@ struct module {
271 /* Startup function. */ 282 /* Startup function. */
272 int (*init)(void); 283 int (*init)(void);
273 284
274 /* If this is non-NULL, vfree after init() returns */ 285 /*
275 void *module_init; 286 * If this is non-NULL, vfree() after init() returns.
287 *
288 * Cacheline align here, such that:
289 * module_init, module_core, init_size, core_size,
290 * init_text_size, core_text_size and mtn_core::{mod,node[0]}
291 * are on the same cacheline.
292 */
293 void *module_init ____cacheline_aligned;
276 294
277 /* Here is the actual code + data, vfree'd on unload. */ 295 /* Here is the actual code + data, vfree'd on unload. */
278 void *module_core; 296 void *module_core;
@@ -283,6 +301,16 @@ struct module {
283 /* The size of the executable code in each section. */ 301 /* The size of the executable code in each section. */
284 unsigned int init_text_size, core_text_size; 302 unsigned int init_text_size, core_text_size;
285 303
304#ifdef CONFIG_MODULES_TREE_LOOKUP
305 /*
306 * We want mtn_core::{mod,node[0]} to be in the same cacheline as the
307 * above entries such that a regular lookup will only touch one
308 * cacheline.
309 */
310 struct mod_tree_node mtn_core;
311 struct mod_tree_node mtn_init;
312#endif
313
286 /* Size of RO sections of the module (text+rodata) */ 314 /* Size of RO sections of the module (text+rodata) */
287 unsigned int init_ro_size, core_ro_size; 315 unsigned int init_ro_size, core_ro_size;
288 316
@@ -369,7 +397,7 @@ struct module {
369 ctor_fn_t *ctors; 397 ctor_fn_t *ctors;
370 unsigned int num_ctors; 398 unsigned int num_ctors;
371#endif 399#endif
372}; 400} ____cacheline_aligned;
373#ifndef MODULE_ARCH_INIT 401#ifndef MODULE_ARCH_INIT
374#define MODULE_ARCH_INIT {} 402#define MODULE_ARCH_INIT {}
375#endif 403#endif
@@ -423,14 +451,22 @@ struct symsearch {
423 bool unused; 451 bool unused;
424}; 452};
425 453
426/* Search for an exported symbol by name. */ 454/*
455 * Search for an exported symbol by name.
456 *
457 * Must be called with module_mutex held or preemption disabled.
458 */
427const struct kernel_symbol *find_symbol(const char *name, 459const struct kernel_symbol *find_symbol(const char *name,
428 struct module **owner, 460 struct module **owner,
429 const unsigned long **crc, 461 const unsigned long **crc,
430 bool gplok, 462 bool gplok,
431 bool warn); 463 bool warn);
432 464
433/* Walk the exported symbol table */ 465/*
466 * Walk the exported symbol table
467 *
468 * Must be called with module_mutex held or preemption disabled.
469 */
434bool each_symbol_section(bool (*fn)(const struct symsearch *arr, 470bool each_symbol_section(bool (*fn)(const struct symsearch *arr,
435 struct module *owner, 471 struct module *owner,
436 void *data), void *data); 472 void *data), void *data);
diff --git a/include/linux/moduleparam.h b/include/linux/moduleparam.h
index 6480dcaca275..c12f2147c350 100644
--- a/include/linux/moduleparam.h
+++ b/include/linux/moduleparam.h
@@ -67,8 +67,9 @@ enum {
67 67
68struct kernel_param { 68struct kernel_param {
69 const char *name; 69 const char *name;
70 struct module *mod;
70 const struct kernel_param_ops *ops; 71 const struct kernel_param_ops *ops;
71 u16 perm; 72 const u16 perm;
72 s8 level; 73 s8 level;
73 u8 flags; 74 u8 flags;
74 union { 75 union {
@@ -108,7 +109,7 @@ struct kparam_array
108 * 109 *
109 * @perm is 0 if the the variable is not to appear in sysfs, or 0444 110 * @perm is 0 if the the variable is not to appear in sysfs, or 0444
110 * for world-readable, 0644 for root-writable, etc. Note that if it 111 * for world-readable, 0644 for root-writable, etc. Note that if it
111 * is writable, you may need to use kparam_block_sysfs_write() around 112 * is writable, you may need to use kernel_param_lock() around
112 * accesses (esp. charp, which can be kfreed when it changes). 113 * accesses (esp. charp, which can be kfreed when it changes).
113 * 114 *
114 * The @type is simply pasted to refer to a param_ops_##type and a 115 * The @type is simply pasted to refer to a param_ops_##type and a
@@ -216,16 +217,16 @@ struct kparam_array
216 parameters. */ 217 parameters. */
217#define __module_param_call(prefix, name, ops, arg, perm, level, flags) \ 218#define __module_param_call(prefix, name, ops, arg, perm, level, flags) \
218 /* Default value instead of permissions? */ \ 219 /* Default value instead of permissions? */ \
219 static const char __param_str_##name[] = prefix #name; \ 220 static const char __param_str_##name[] = prefix #name; \
220 static struct kernel_param __moduleparam_const __param_##name \ 221 static struct kernel_param __moduleparam_const __param_##name \
221 __used \ 222 __used \
222 __attribute__ ((unused,__section__ ("__param"),aligned(sizeof(void *)))) \ 223 __attribute__ ((unused,__section__ ("__param"),aligned(sizeof(void *)))) \
223 = { __param_str_##name, ops, VERIFY_OCTAL_PERMISSIONS(perm), \ 224 = { __param_str_##name, THIS_MODULE, ops, \
224 level, flags, { arg } } 225 VERIFY_OCTAL_PERMISSIONS(perm), level, flags, { arg } }
225 226
226/* Obsolete - use module_param_cb() */ 227/* Obsolete - use module_param_cb() */
227#define module_param_call(name, set, get, arg, perm) \ 228#define module_param_call(name, set, get, arg, perm) \
228 static struct kernel_param_ops __param_ops_##name = \ 229 static const struct kernel_param_ops __param_ops_##name = \
229 { .flags = 0, (void *)set, (void *)get }; \ 230 { .flags = 0, (void *)set, (void *)get }; \
230 __module_param_call(MODULE_PARAM_PREFIX, \ 231 __module_param_call(MODULE_PARAM_PREFIX, \
231 name, &__param_ops_##name, arg, \ 232 name, &__param_ops_##name, arg, \
@@ -238,58 +239,14 @@ __check_old_set_param(int (*oldset)(const char *, struct kernel_param *))
238 return 0; 239 return 0;
239} 240}
240 241
241/**
242 * kparam_block_sysfs_write - make sure a parameter isn't written via sysfs.
243 * @name: the name of the parameter
244 *
245 * There's no point blocking write on a paramter that isn't writable via sysfs!
246 */
247#define kparam_block_sysfs_write(name) \
248 do { \
249 BUG_ON(!(__param_##name.perm & 0222)); \
250 __kernel_param_lock(); \
251 } while (0)
252
253/**
254 * kparam_unblock_sysfs_write - allows sysfs to write to a parameter again.
255 * @name: the name of the parameter
256 */
257#define kparam_unblock_sysfs_write(name) \
258 do { \
259 BUG_ON(!(__param_##name.perm & 0222)); \
260 __kernel_param_unlock(); \
261 } while (0)
262
263/**
264 * kparam_block_sysfs_read - make sure a parameter isn't read via sysfs.
265 * @name: the name of the parameter
266 *
267 * This also blocks sysfs writes.
268 */
269#define kparam_block_sysfs_read(name) \
270 do { \
271 BUG_ON(!(__param_##name.perm & 0444)); \
272 __kernel_param_lock(); \
273 } while (0)
274
275/**
276 * kparam_unblock_sysfs_read - allows sysfs to read a parameter again.
277 * @name: the name of the parameter
278 */
279#define kparam_unblock_sysfs_read(name) \
280 do { \
281 BUG_ON(!(__param_##name.perm & 0444)); \
282 __kernel_param_unlock(); \
283 } while (0)
284
285#ifdef CONFIG_SYSFS 242#ifdef CONFIG_SYSFS
286extern void __kernel_param_lock(void); 243extern void kernel_param_lock(struct module *mod);
287extern void __kernel_param_unlock(void); 244extern void kernel_param_unlock(struct module *mod);
288#else 245#else
289static inline void __kernel_param_lock(void) 246static inline void kernel_param_lock(struct module *mod)
290{ 247{
291} 248}
292static inline void __kernel_param_unlock(void) 249static inline void kernel_param_unlock(struct module *mod)
293{ 250{
294} 251}
295#endif 252#endif
@@ -386,64 +343,70 @@ static inline void destroy_params(const struct kernel_param *params,
386#define __param_check(name, p, type) \ 343#define __param_check(name, p, type) \
387 static inline type __always_unused *__check_##name(void) { return(p); } 344 static inline type __always_unused *__check_##name(void) { return(p); }
388 345
389extern struct kernel_param_ops param_ops_byte; 346extern const struct kernel_param_ops param_ops_byte;
390extern int param_set_byte(const char *val, const struct kernel_param *kp); 347extern int param_set_byte(const char *val, const struct kernel_param *kp);
391extern int param_get_byte(char *buffer, const struct kernel_param *kp); 348extern int param_get_byte(char *buffer, const struct kernel_param *kp);
392#define param_check_byte(name, p) __param_check(name, p, unsigned char) 349#define param_check_byte(name, p) __param_check(name, p, unsigned char)
393 350
394extern struct kernel_param_ops param_ops_short; 351extern const struct kernel_param_ops param_ops_short;
395extern int param_set_short(const char *val, const struct kernel_param *kp); 352extern int param_set_short(const char *val, const struct kernel_param *kp);
396extern int param_get_short(char *buffer, const struct kernel_param *kp); 353extern int param_get_short(char *buffer, const struct kernel_param *kp);
397#define param_check_short(name, p) __param_check(name, p, short) 354#define param_check_short(name, p) __param_check(name, p, short)
398 355
399extern struct kernel_param_ops param_ops_ushort; 356extern const struct kernel_param_ops param_ops_ushort;
400extern int param_set_ushort(const char *val, const struct kernel_param *kp); 357extern int param_set_ushort(const char *val, const struct kernel_param *kp);
401extern int param_get_ushort(char *buffer, const struct kernel_param *kp); 358extern int param_get_ushort(char *buffer, const struct kernel_param *kp);
402#define param_check_ushort(name, p) __param_check(name, p, unsigned short) 359#define param_check_ushort(name, p) __param_check(name, p, unsigned short)
403 360
404extern struct kernel_param_ops param_ops_int; 361extern const struct kernel_param_ops param_ops_int;
405extern int param_set_int(const char *val, const struct kernel_param *kp); 362extern int param_set_int(const char *val, const struct kernel_param *kp);
406extern int param_get_int(char *buffer, const struct kernel_param *kp); 363extern int param_get_int(char *buffer, const struct kernel_param *kp);
407#define param_check_int(name, p) __param_check(name, p, int) 364#define param_check_int(name, p) __param_check(name, p, int)
408 365
409extern struct kernel_param_ops param_ops_uint; 366extern const struct kernel_param_ops param_ops_uint;
410extern int param_set_uint(const char *val, const struct kernel_param *kp); 367extern int param_set_uint(const char *val, const struct kernel_param *kp);
411extern int param_get_uint(char *buffer, const struct kernel_param *kp); 368extern int param_get_uint(char *buffer, const struct kernel_param *kp);
412#define param_check_uint(name, p) __param_check(name, p, unsigned int) 369#define param_check_uint(name, p) __param_check(name, p, unsigned int)
413 370
414extern struct kernel_param_ops param_ops_long; 371extern const struct kernel_param_ops param_ops_long;
415extern int param_set_long(const char *val, const struct kernel_param *kp); 372extern int param_set_long(const char *val, const struct kernel_param *kp);
416extern int param_get_long(char *buffer, const struct kernel_param *kp); 373extern int param_get_long(char *buffer, const struct kernel_param *kp);
417#define param_check_long(name, p) __param_check(name, p, long) 374#define param_check_long(name, p) __param_check(name, p, long)
418 375
419extern struct kernel_param_ops param_ops_ulong; 376extern const struct kernel_param_ops param_ops_ulong;
420extern int param_set_ulong(const char *val, const struct kernel_param *kp); 377extern int param_set_ulong(const char *val, const struct kernel_param *kp);
421extern int param_get_ulong(char *buffer, const struct kernel_param *kp); 378extern int param_get_ulong(char *buffer, const struct kernel_param *kp);
422#define param_check_ulong(name, p) __param_check(name, p, unsigned long) 379#define param_check_ulong(name, p) __param_check(name, p, unsigned long)
423 380
424extern struct kernel_param_ops param_ops_ullong; 381extern const struct kernel_param_ops param_ops_ullong;
425extern int param_set_ullong(const char *val, const struct kernel_param *kp); 382extern int param_set_ullong(const char *val, const struct kernel_param *kp);
426extern int param_get_ullong(char *buffer, const struct kernel_param *kp); 383extern int param_get_ullong(char *buffer, const struct kernel_param *kp);
427#define param_check_ullong(name, p) __param_check(name, p, unsigned long long) 384#define param_check_ullong(name, p) __param_check(name, p, unsigned long long)
428 385
429extern struct kernel_param_ops param_ops_charp; 386extern const struct kernel_param_ops param_ops_charp;
430extern int param_set_charp(const char *val, const struct kernel_param *kp); 387extern int param_set_charp(const char *val, const struct kernel_param *kp);
431extern int param_get_charp(char *buffer, const struct kernel_param *kp); 388extern int param_get_charp(char *buffer, const struct kernel_param *kp);
432#define param_check_charp(name, p) __param_check(name, p, char *) 389#define param_check_charp(name, p) __param_check(name, p, char *)
433 390
434/* We used to allow int as well as bool. We're taking that away! */ 391/* We used to allow int as well as bool. We're taking that away! */
435extern struct kernel_param_ops param_ops_bool; 392extern const struct kernel_param_ops param_ops_bool;
436extern int param_set_bool(const char *val, const struct kernel_param *kp); 393extern int param_set_bool(const char *val, const struct kernel_param *kp);
437extern int param_get_bool(char *buffer, const struct kernel_param *kp); 394extern int param_get_bool(char *buffer, const struct kernel_param *kp);
438#define param_check_bool(name, p) __param_check(name, p, bool) 395#define param_check_bool(name, p) __param_check(name, p, bool)
439 396
440extern struct kernel_param_ops param_ops_invbool; 397extern const struct kernel_param_ops param_ops_bool_enable_only;
398extern int param_set_bool_enable_only(const char *val,
399 const struct kernel_param *kp);
400/* getter is the same as for the regular bool */
401#define param_check_bool_enable_only param_check_bool
402
403extern const struct kernel_param_ops param_ops_invbool;
441extern int param_set_invbool(const char *val, const struct kernel_param *kp); 404extern int param_set_invbool(const char *val, const struct kernel_param *kp);
442extern int param_get_invbool(char *buffer, const struct kernel_param *kp); 405extern int param_get_invbool(char *buffer, const struct kernel_param *kp);
443#define param_check_invbool(name, p) __param_check(name, p, bool) 406#define param_check_invbool(name, p) __param_check(name, p, bool)
444 407
445/* An int, which can only be set like a bool (though it shows as an int). */ 408/* An int, which can only be set like a bool (though it shows as an int). */
446extern struct kernel_param_ops param_ops_bint; 409extern const struct kernel_param_ops param_ops_bint;
447extern int param_set_bint(const char *val, const struct kernel_param *kp); 410extern int param_set_bint(const char *val, const struct kernel_param *kp);
448#define param_get_bint param_get_int 411#define param_get_bint param_get_int
449#define param_check_bint param_check_int 412#define param_check_bint param_check_int
@@ -487,9 +450,9 @@ extern int param_set_bint(const char *val, const struct kernel_param *kp);
487 perm, -1, 0); \ 450 perm, -1, 0); \
488 __MODULE_PARM_TYPE(name, "array of " #type) 451 __MODULE_PARM_TYPE(name, "array of " #type)
489 452
490extern struct kernel_param_ops param_array_ops; 453extern const struct kernel_param_ops param_array_ops;
491 454
492extern struct kernel_param_ops param_ops_string; 455extern const struct kernel_param_ops param_ops_string;
493extern int param_set_copystring(const char *val, const struct kernel_param *); 456extern int param_set_copystring(const char *val, const struct kernel_param *);
494extern int param_get_string(char *buffer, const struct kernel_param *kp); 457extern int param_get_string(char *buffer, const struct kernel_param *kp);
495 458
diff --git a/include/linux/rbtree.h b/include/linux/rbtree.h
index fb31765e935a..830c4992088d 100644
--- a/include/linux/rbtree.h
+++ b/include/linux/rbtree.h
@@ -31,6 +31,7 @@
31 31
32#include <linux/kernel.h> 32#include <linux/kernel.h>
33#include <linux/stddef.h> 33#include <linux/stddef.h>
34#include <linux/rcupdate.h>
34 35
35struct rb_node { 36struct rb_node {
36 unsigned long __rb_parent_color; 37 unsigned long __rb_parent_color;
@@ -73,11 +74,11 @@ extern struct rb_node *rb_first_postorder(const struct rb_root *);
73extern struct rb_node *rb_next_postorder(const struct rb_node *); 74extern struct rb_node *rb_next_postorder(const struct rb_node *);
74 75
75/* Fast replacement of a single node without remove/rebalance/add/rebalance */ 76/* Fast replacement of a single node without remove/rebalance/add/rebalance */
76extern void rb_replace_node(struct rb_node *victim, struct rb_node *new, 77extern void rb_replace_node(struct rb_node *victim, struct rb_node *new,
77 struct rb_root *root); 78 struct rb_root *root);
78 79
79static inline void rb_link_node(struct rb_node * node, struct rb_node * parent, 80static inline void rb_link_node(struct rb_node *node, struct rb_node *parent,
80 struct rb_node ** rb_link) 81 struct rb_node **rb_link)
81{ 82{
82 node->__rb_parent_color = (unsigned long)parent; 83 node->__rb_parent_color = (unsigned long)parent;
83 node->rb_left = node->rb_right = NULL; 84 node->rb_left = node->rb_right = NULL;
@@ -85,6 +86,15 @@ static inline void rb_link_node(struct rb_node * node, struct rb_node * parent,
85 *rb_link = node; 86 *rb_link = node;
86} 87}
87 88
89static inline void rb_link_node_rcu(struct rb_node *node, struct rb_node *parent,
90 struct rb_node **rb_link)
91{
92 node->__rb_parent_color = (unsigned long)parent;
93 node->rb_left = node->rb_right = NULL;
94
95 rcu_assign_pointer(*rb_link, node);
96}
97
88#define rb_entry_safe(ptr, type, member) \ 98#define rb_entry_safe(ptr, type, member) \
89 ({ typeof(ptr) ____ptr = (ptr); \ 99 ({ typeof(ptr) ____ptr = (ptr); \
90 ____ptr ? rb_entry(____ptr, type, member) : NULL; \ 100 ____ptr ? rb_entry(____ptr, type, member) : NULL; \
diff --git a/include/linux/rbtree_augmented.h b/include/linux/rbtree_augmented.h
index 378c5ee75f78..14d7b831b63a 100644
--- a/include/linux/rbtree_augmented.h
+++ b/include/linux/rbtree_augmented.h
@@ -123,11 +123,11 @@ __rb_change_child(struct rb_node *old, struct rb_node *new,
123{ 123{
124 if (parent) { 124 if (parent) {
125 if (parent->rb_left == old) 125 if (parent->rb_left == old)
126 parent->rb_left = new; 126 WRITE_ONCE(parent->rb_left, new);
127 else 127 else
128 parent->rb_right = new; 128 WRITE_ONCE(parent->rb_right, new);
129 } else 129 } else
130 root->rb_node = new; 130 WRITE_ONCE(root->rb_node, new);
131} 131}
132 132
133extern void __rb_erase_color(struct rb_node *parent, struct rb_root *root, 133extern void __rb_erase_color(struct rb_node *parent, struct rb_root *root,
@@ -137,7 +137,8 @@ static __always_inline struct rb_node *
137__rb_erase_augmented(struct rb_node *node, struct rb_root *root, 137__rb_erase_augmented(struct rb_node *node, struct rb_root *root,
138 const struct rb_augment_callbacks *augment) 138 const struct rb_augment_callbacks *augment)
139{ 139{
140 struct rb_node *child = node->rb_right, *tmp = node->rb_left; 140 struct rb_node *child = node->rb_right;
141 struct rb_node *tmp = node->rb_left;
141 struct rb_node *parent, *rebalance; 142 struct rb_node *parent, *rebalance;
142 unsigned long pc; 143 unsigned long pc;
143 144
@@ -167,6 +168,7 @@ __rb_erase_augmented(struct rb_node *node, struct rb_root *root,
167 tmp = parent; 168 tmp = parent;
168 } else { 169 } else {
169 struct rb_node *successor = child, *child2; 170 struct rb_node *successor = child, *child2;
171
170 tmp = child->rb_left; 172 tmp = child->rb_left;
171 if (!tmp) { 173 if (!tmp) {
172 /* 174 /*
@@ -180,6 +182,7 @@ __rb_erase_augmented(struct rb_node *node, struct rb_root *root,
180 */ 182 */
181 parent = successor; 183 parent = successor;
182 child2 = successor->rb_right; 184 child2 = successor->rb_right;
185
183 augment->copy(node, successor); 186 augment->copy(node, successor);
184 } else { 187 } else {
185 /* 188 /*
@@ -201,19 +204,23 @@ __rb_erase_augmented(struct rb_node *node, struct rb_root *root,
201 successor = tmp; 204 successor = tmp;
202 tmp = tmp->rb_left; 205 tmp = tmp->rb_left;
203 } while (tmp); 206 } while (tmp);
204 parent->rb_left = child2 = successor->rb_right; 207 child2 = successor->rb_right;
205 successor->rb_right = child; 208 WRITE_ONCE(parent->rb_left, child2);
209 WRITE_ONCE(successor->rb_right, child);
206 rb_set_parent(child, successor); 210 rb_set_parent(child, successor);
211
207 augment->copy(node, successor); 212 augment->copy(node, successor);
208 augment->propagate(parent, successor); 213 augment->propagate(parent, successor);
209 } 214 }
210 215
211 successor->rb_left = tmp = node->rb_left; 216 tmp = node->rb_left;
217 WRITE_ONCE(successor->rb_left, tmp);
212 rb_set_parent(tmp, successor); 218 rb_set_parent(tmp, successor);
213 219
214 pc = node->__rb_parent_color; 220 pc = node->__rb_parent_color;
215 tmp = __rb_parent(pc); 221 tmp = __rb_parent(pc);
216 __rb_change_child(node, successor, tmp, root); 222 __rb_change_child(node, successor, tmp, root);
223
217 if (child2) { 224 if (child2) {
218 successor->__rb_parent_color = pc; 225 successor->__rb_parent_color = pc;
219 rb_set_parent_color(child2, parent, RB_BLACK); 226 rb_set_parent_color(child2, parent, RB_BLACK);
diff --git a/include/linux/rbtree_latch.h b/include/linux/rbtree_latch.h
new file mode 100644
index 000000000000..4f3432c61d12
--- /dev/null
+++ b/include/linux/rbtree_latch.h
@@ -0,0 +1,212 @@
1/*
2 * Latched RB-trees
3 *
4 * Copyright (C) 2015 Intel Corp., Peter Zijlstra <peterz@infradead.org>
5 *
6 * Since RB-trees have non-atomic modifications they're not immediately suited
7 * for RCU/lockless queries. Even though we made RB-tree lookups non-fatal for
8 * lockless lookups; we cannot guarantee they return a correct result.
9 *
10 * The simplest solution is a seqlock + RB-tree, this will allow lockless
11 * lookups; but has the constraint (inherent to the seqlock) that read sides
12 * cannot nest in write sides.
13 *
14 * If we need to allow unconditional lookups (say as required for NMI context
15 * usage) we need a more complex setup; this data structure provides this by
16 * employing the latch technique -- see @raw_write_seqcount_latch -- to
17 * implement a latched RB-tree which does allow for unconditional lookups by
18 * virtue of always having (at least) one stable copy of the tree.
19 *
20 * However, while we have the guarantee that there is at all times one stable
21 * copy, this does not guarantee an iteration will not observe modifications.
22 * What might have been a stable copy at the start of the iteration, need not
23 * remain so for the duration of the iteration.
24 *
25 * Therefore, this does require a lockless RB-tree iteration to be non-fatal;
26 * see the comment in lib/rbtree.c. Note however that we only require the first
27 * condition -- not seeing partial stores -- because the latch thing isolates
28 * us from loops. If we were to interrupt a modification the lookup would be
29 * pointed at the stable tree and complete while the modification was halted.
30 */
31
32#ifndef RB_TREE_LATCH_H
33#define RB_TREE_LATCH_H
34
35#include <linux/rbtree.h>
36#include <linux/seqlock.h>
37
38struct latch_tree_node {
39 struct rb_node node[2];
40};
41
42struct latch_tree_root {
43 seqcount_t seq;
44 struct rb_root tree[2];
45};
46
47/**
48 * latch_tree_ops - operators to define the tree order
49 * @less: used for insertion; provides the (partial) order between two elements.
50 * @comp: used for lookups; provides the order between the search key and an element.
51 *
52 * The operators are related like:
53 *
54 * comp(a->key,b) < 0 := less(a,b)
55 * comp(a->key,b) > 0 := less(b,a)
56 * comp(a->key,b) == 0 := !less(a,b) && !less(b,a)
57 *
58 * If these operators define a partial order on the elements we make no
59 * guarantee on which of the elements matching the key is found. See
60 * latch_tree_find().
61 */
62struct latch_tree_ops {
63 bool (*less)(struct latch_tree_node *a, struct latch_tree_node *b);
64 int (*comp)(void *key, struct latch_tree_node *b);
65};
66
67static __always_inline struct latch_tree_node *
68__lt_from_rb(struct rb_node *node, int idx)
69{
70 return container_of(node, struct latch_tree_node, node[idx]);
71}
72
73static __always_inline void
74__lt_insert(struct latch_tree_node *ltn, struct latch_tree_root *ltr, int idx,
75 bool (*less)(struct latch_tree_node *a, struct latch_tree_node *b))
76{
77 struct rb_root *root = &ltr->tree[idx];
78 struct rb_node **link = &root->rb_node;
79 struct rb_node *node = &ltn->node[idx];
80 struct rb_node *parent = NULL;
81 struct latch_tree_node *ltp;
82
83 while (*link) {
84 parent = *link;
85 ltp = __lt_from_rb(parent, idx);
86
87 if (less(ltn, ltp))
88 link = &parent->rb_left;
89 else
90 link = &parent->rb_right;
91 }
92
93 rb_link_node_rcu(node, parent, link);
94 rb_insert_color(node, root);
95}
96
97static __always_inline void
98__lt_erase(struct latch_tree_node *ltn, struct latch_tree_root *ltr, int idx)
99{
100 rb_erase(&ltn->node[idx], &ltr->tree[idx]);
101}
102
103static __always_inline struct latch_tree_node *
104__lt_find(void *key, struct latch_tree_root *ltr, int idx,
105 int (*comp)(void *key, struct latch_tree_node *node))
106{
107 struct rb_node *node = rcu_dereference_raw(ltr->tree[idx].rb_node);
108 struct latch_tree_node *ltn;
109 int c;
110
111 while (node) {
112 ltn = __lt_from_rb(node, idx);
113 c = comp(key, ltn);
114
115 if (c < 0)
116 node = rcu_dereference_raw(node->rb_left);
117 else if (c > 0)
118 node = rcu_dereference_raw(node->rb_right);
119 else
120 return ltn;
121 }
122
123 return NULL;
124}
125
126/**
127 * latch_tree_insert() - insert @node into the trees @root
128 * @node: nodes to insert
129 * @root: trees to insert @node into
130 * @ops: operators defining the node order
131 *
132 * It inserts @node into @root in an ordered fashion such that we can always
133 * observe one complete tree. See the comment for raw_write_seqcount_latch().
134 *
135 * The inserts use rcu_assign_pointer() to publish the element such that the
136 * tree structure is stored before we can observe the new @node.
137 *
138 * All modifications (latch_tree_insert, latch_tree_remove) are assumed to be
139 * serialized.
140 */
141static __always_inline void
142latch_tree_insert(struct latch_tree_node *node,
143 struct latch_tree_root *root,
144 const struct latch_tree_ops *ops)
145{
146 raw_write_seqcount_latch(&root->seq);
147 __lt_insert(node, root, 0, ops->less);
148 raw_write_seqcount_latch(&root->seq);
149 __lt_insert(node, root, 1, ops->less);
150}
151
152/**
153 * latch_tree_erase() - removes @node from the trees @root
154 * @node: nodes to remote
155 * @root: trees to remove @node from
156 * @ops: operators defining the node order
157 *
158 * Removes @node from the trees @root in an ordered fashion such that we can
159 * always observe one complete tree. See the comment for
160 * raw_write_seqcount_latch().
161 *
162 * It is assumed that @node will observe one RCU quiescent state before being
163 * reused of freed.
164 *
165 * All modifications (latch_tree_insert, latch_tree_remove) are assumed to be
166 * serialized.
167 */
168static __always_inline void
169latch_tree_erase(struct latch_tree_node *node,
170 struct latch_tree_root *root,
171 const struct latch_tree_ops *ops)
172{
173 raw_write_seqcount_latch(&root->seq);
174 __lt_erase(node, root, 0);
175 raw_write_seqcount_latch(&root->seq);
176 __lt_erase(node, root, 1);
177}
178
179/**
180 * latch_tree_find() - find the node matching @key in the trees @root
181 * @key: search key
182 * @root: trees to search for @key
183 * @ops: operators defining the node order
184 *
185 * Does a lockless lookup in the trees @root for the node matching @key.
186 *
187 * It is assumed that this is called while holding the appropriate RCU read
188 * side lock.
189 *
190 * If the operators define a partial order on the elements (there are multiple
191 * elements which have the same key value) it is undefined which of these
192 * elements will be found. Nor is it possible to iterate the tree to find
193 * further elements with the same key value.
194 *
195 * Returns: a pointer to the node matching @key or NULL.
196 */
197static __always_inline struct latch_tree_node *
198latch_tree_find(void *key, struct latch_tree_root *root,
199 const struct latch_tree_ops *ops)
200{
201 struct latch_tree_node *node;
202 unsigned int seq;
203
204 do {
205 seq = raw_read_seqcount_latch(&root->seq);
206 node = __lt_find(key, root, seq & 1, ops->comp);
207 } while (read_seqcount_retry(&root->seq, seq));
208
209 return node;
210}
211
212#endif /* RB_TREE_LATCH_H */
diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h
index 33a056bb886f..4cf5f51b4c9c 100644
--- a/include/linux/rcupdate.h
+++ b/include/linux/rcupdate.h
@@ -633,21 +633,6 @@ static inline void rcu_preempt_sleep_check(void)
633#define RCU_INITIALIZER(v) (typeof(*(v)) __force __rcu *)(v) 633#define RCU_INITIALIZER(v) (typeof(*(v)) __force __rcu *)(v)
634 634
635/** 635/**
636 * lockless_dereference() - safely load a pointer for later dereference
637 * @p: The pointer to load
638 *
639 * Similar to rcu_dereference(), but for situations where the pointed-to
640 * object's lifetime is managed by something other than RCU. That
641 * "something other" might be reference counting or simple immortality.
642 */
643#define lockless_dereference(p) \
644({ \
645 typeof(p) _________p1 = READ_ONCE(p); \
646 smp_read_barrier_depends(); /* Dependency order vs. p above. */ \
647 (_________p1); \
648})
649
650/**
651 * rcu_assign_pointer() - assign to RCU-protected pointer 636 * rcu_assign_pointer() - assign to RCU-protected pointer
652 * @p: pointer to assign to 637 * @p: pointer to assign to
653 * @v: value to assign (publish) 638 * @v: value to assign (publish)
diff --git a/include/linux/seqlock.h b/include/linux/seqlock.h
index 486e685a226a..e0582106ef4f 100644
--- a/include/linux/seqlock.h
+++ b/include/linux/seqlock.h
@@ -35,6 +35,7 @@
35#include <linux/spinlock.h> 35#include <linux/spinlock.h>
36#include <linux/preempt.h> 36#include <linux/preempt.h>
37#include <linux/lockdep.h> 37#include <linux/lockdep.h>
38#include <linux/compiler.h>
38#include <asm/processor.h> 39#include <asm/processor.h>
39 40
40/* 41/*
@@ -274,9 +275,87 @@ static inline void raw_write_seqcount_barrier(seqcount_t *s)
274 s->sequence++; 275 s->sequence++;
275} 276}
276 277
277/* 278static inline int raw_read_seqcount_latch(seqcount_t *s)
279{
280 return lockless_dereference(s->sequence);
281}
282
283/**
278 * raw_write_seqcount_latch - redirect readers to even/odd copy 284 * raw_write_seqcount_latch - redirect readers to even/odd copy
279 * @s: pointer to seqcount_t 285 * @s: pointer to seqcount_t
286 *
287 * The latch technique is a multiversion concurrency control method that allows
288 * queries during non-atomic modifications. If you can guarantee queries never
289 * interrupt the modification -- e.g. the concurrency is strictly between CPUs
290 * -- you most likely do not need this.
291 *
292 * Where the traditional RCU/lockless data structures rely on atomic
293 * modifications to ensure queries observe either the old or the new state the
294 * latch allows the same for non-atomic updates. The trade-off is doubling the
295 * cost of storage; we have to maintain two copies of the entire data
296 * structure.
297 *
298 * Very simply put: we first modify one copy and then the other. This ensures
299 * there is always one copy in a stable state, ready to give us an answer.
300 *
301 * The basic form is a data structure like:
302 *
303 * struct latch_struct {
304 * seqcount_t seq;
305 * struct data_struct data[2];
306 * };
307 *
308 * Where a modification, which is assumed to be externally serialized, does the
309 * following:
310 *
311 * void latch_modify(struct latch_struct *latch, ...)
312 * {
313 * smp_wmb(); <- Ensure that the last data[1] update is visible
314 * latch->seq++;
315 * smp_wmb(); <- Ensure that the seqcount update is visible
316 *
317 * modify(latch->data[0], ...);
318 *
319 * smp_wmb(); <- Ensure that the data[0] update is visible
320 * latch->seq++;
321 * smp_wmb(); <- Ensure that the seqcount update is visible
322 *
323 * modify(latch->data[1], ...);
324 * }
325 *
326 * The query will have a form like:
327 *
328 * struct entry *latch_query(struct latch_struct *latch, ...)
329 * {
330 * struct entry *entry;
331 * unsigned seq, idx;
332 *
333 * do {
334 * seq = lockless_dereference(latch->seq);
335 *
336 * idx = seq & 0x01;
337 * entry = data_query(latch->data[idx], ...);
338 *
339 * smp_rmb();
340 * } while (seq != latch->seq);
341 *
342 * return entry;
343 * }
344 *
345 * So during the modification, queries are first redirected to data[1]. Then we
346 * modify data[0]. When that is complete, we redirect queries back to data[0]
347 * and we can modify data[1].
348 *
349 * NOTE: The non-requirement for atomic modifications does _NOT_ include
350 * the publishing of new entries in the case where data is a dynamic
351 * data structure.
352 *
353 * An iteration might start in data[0] and get suspended long enough
354 * to miss an entire modification sequence, once it resumes it might
355 * observe the new entry.
356 *
357 * NOTE: When data is a dynamic data structure; one should use regular RCU
358 * patterns to manage the lifetimes of the objects within.
280 */ 359 */
281static inline void raw_write_seqcount_latch(seqcount_t *s) 360static inline void raw_write_seqcount_latch(seqcount_t *s)
282{ 361{
diff --git a/init/Kconfig b/init/Kconfig
index 7d1ffd2ae536..bcc41bd19999 100644
--- a/init/Kconfig
+++ b/init/Kconfig
@@ -1941,26 +1941,21 @@ config MODULE_COMPRESS
1941 bool "Compress modules on installation" 1941 bool "Compress modules on installation"
1942 depends on MODULES 1942 depends on MODULES
1943 help 1943 help
1944 This option compresses the kernel modules when 'make
1945 modules_install' is run.
1946 1944
1947 The modules will be compressed either using gzip or xz depend on the 1945 Compresses kernel modules when 'make modules_install' is run; gzip or
1948 choice made in "Compression algorithm". 1946 xz depending on "Compression algorithm" below.
1949 1947
1950 module-init-tools has support for gzip format while kmod handle gzip 1948 module-init-tools MAY support gzip, and kmod MAY support gzip and xz.
1951 and xz compressed modules.
1952 1949
1953 When a kernel module is installed from outside of the main kernel 1950 Out-of-tree kernel modules installed using Kbuild will also be
1954 source and uses the Kbuild system for installing modules then that 1951 compressed upon installation.
1955 kernel module will also be compressed when it is installed.
1956 1952
1957 This option provides little benefit when the modules are to be used inside 1953 Note: for modules inside an initrd or initramfs, it's more efficient
1958 an initrd or initramfs, it generally is more efficient to compress the whole 1954 to compress the whole initrd or initramfs instead.
1959 initrd or initramfs instead.
1960 1955
1961 This is fully compatible with signed modules while the signed module is 1956 Note: This is fully compatible with signed modules.
1962 compressed. module-init-tools or kmod handles decompression and provide to 1957
1963 other layer the uncompressed but signed payload. 1958 If in doubt, say N.
1964 1959
1965choice 1960choice
1966 prompt "Compression algorithm" 1961 prompt "Compression algorithm"
@@ -1982,6 +1977,10 @@ endchoice
1982 1977
1983endif # MODULES 1978endif # MODULES
1984 1979
1980config MODULES_TREE_LOOKUP
1981 def_bool y
1982 depends on PERF_EVENTS || TRACING
1983
1985config INIT_ALL_POSSIBLE 1984config INIT_ALL_POSSIBLE
1986 bool 1985 bool
1987 help 1986 help
diff --git a/kernel/jump_label.c b/kernel/jump_label.c
index 9019f15deab2..52ebaca1b9fc 100644
--- a/kernel/jump_label.c
+++ b/kernel/jump_label.c
@@ -302,7 +302,7 @@ static int jump_label_add_module(struct module *mod)
302 continue; 302 continue;
303 303
304 key = iterk; 304 key = iterk;
305 if (__module_address(iter->key) == mod) { 305 if (within_module(iter->key, mod)) {
306 /* 306 /*
307 * Set key->entries to iter, but preserve JUMP_LABEL_TRUE_BRANCH. 307 * Set key->entries to iter, but preserve JUMP_LABEL_TRUE_BRANCH.
308 */ 308 */
@@ -339,7 +339,7 @@ static void jump_label_del_module(struct module *mod)
339 339
340 key = (struct static_key *)(unsigned long)iter->key; 340 key = (struct static_key *)(unsigned long)iter->key;
341 341
342 if (__module_address(iter->key) == mod) 342 if (within_module(iter->key, mod))
343 continue; 343 continue;
344 344
345 prev = &key->next; 345 prev = &key->next;
@@ -443,14 +443,16 @@ static void jump_label_update(struct static_key *key, int enable)
443{ 443{
444 struct jump_entry *stop = __stop___jump_table; 444 struct jump_entry *stop = __stop___jump_table;
445 struct jump_entry *entry = jump_label_get_entries(key); 445 struct jump_entry *entry = jump_label_get_entries(key);
446
447#ifdef CONFIG_MODULES 446#ifdef CONFIG_MODULES
448 struct module *mod = __module_address((unsigned long)key); 447 struct module *mod;
449 448
450 __jump_label_mod_update(key, enable); 449 __jump_label_mod_update(key, enable);
451 450
451 preempt_disable();
452 mod = __module_address((unsigned long)key);
452 if (mod) 453 if (mod)
453 stop = mod->jump_entries + mod->num_jump_entries; 454 stop = mod->jump_entries + mod->num_jump_entries;
455 preempt_enable();
454#endif 456#endif
455 /* if there are no users, entry can be NULL */ 457 /* if there are no users, entry can be NULL */
456 if (entry) 458 if (entry)
diff --git a/kernel/module.c b/kernel/module.c
index f80a97f7da1f..3e0e19763d24 100644
--- a/kernel/module.c
+++ b/kernel/module.c
@@ -101,48 +101,201 @@
101DEFINE_MUTEX(module_mutex); 101DEFINE_MUTEX(module_mutex);
102EXPORT_SYMBOL_GPL(module_mutex); 102EXPORT_SYMBOL_GPL(module_mutex);
103static LIST_HEAD(modules); 103static LIST_HEAD(modules);
104#ifdef CONFIG_KGDB_KDB
105struct list_head *kdb_modules = &modules; /* kdb needs the list of modules */
106#endif /* CONFIG_KGDB_KDB */
107 104
108#ifdef CONFIG_MODULE_SIG 105#ifdef CONFIG_MODULES_TREE_LOOKUP
109#ifdef CONFIG_MODULE_SIG_FORCE 106
110static bool sig_enforce = true; 107/*
111#else 108 * Use a latched RB-tree for __module_address(); this allows us to use
112static bool sig_enforce = false; 109 * RCU-sched lookups of the address from any context.
110 *
111 * Because modules have two address ranges: init and core, we need two
112 * latch_tree_nodes entries. Therefore we need the back-pointer from
113 * mod_tree_node.
114 *
115 * Because init ranges are short lived we mark them unlikely and have placed
116 * them outside the critical cacheline in struct module.
117 *
118 * This is conditional on PERF_EVENTS || TRACING because those can really hit
119 * __module_address() hard by doing a lot of stack unwinding; potentially from
120 * NMI context.
121 */
113 122
114static int param_set_bool_enable_only(const char *val, 123static __always_inline unsigned long __mod_tree_val(struct latch_tree_node *n)
115 const struct kernel_param *kp)
116{ 124{
117 int err; 125 struct mod_tree_node *mtn = container_of(n, struct mod_tree_node, node);
118 bool test; 126 struct module *mod = mtn->mod;
119 struct kernel_param dummy_kp = *kp;
120 127
121 dummy_kp.arg = &test; 128 if (unlikely(mtn == &mod->mtn_init))
129 return (unsigned long)mod->module_init;
122 130
123 err = param_set_bool(val, &dummy_kp); 131 return (unsigned long)mod->module_core;
124 if (err) 132}
125 return err; 133
134static __always_inline unsigned long __mod_tree_size(struct latch_tree_node *n)
135{
136 struct mod_tree_node *mtn = container_of(n, struct mod_tree_node, node);
137 struct module *mod = mtn->mod;
126 138
127 /* Don't let them unset it once it's set! */ 139 if (unlikely(mtn == &mod->mtn_init))
128 if (!test && sig_enforce) 140 return (unsigned long)mod->init_size;
129 return -EROFS; 141
142 return (unsigned long)mod->core_size;
143}
144
145static __always_inline bool
146mod_tree_less(struct latch_tree_node *a, struct latch_tree_node *b)
147{
148 return __mod_tree_val(a) < __mod_tree_val(b);
149}
150
151static __always_inline int
152mod_tree_comp(void *key, struct latch_tree_node *n)
153{
154 unsigned long val = (unsigned long)key;
155 unsigned long start, end;
156
157 start = __mod_tree_val(n);
158 if (val < start)
159 return -1;
160
161 end = start + __mod_tree_size(n);
162 if (val >= end)
163 return 1;
130 164
131 if (test)
132 sig_enforce = true;
133 return 0; 165 return 0;
134} 166}
135 167
136static const struct kernel_param_ops param_ops_bool_enable_only = { 168static const struct latch_tree_ops mod_tree_ops = {
137 .flags = KERNEL_PARAM_OPS_FL_NOARG, 169 .less = mod_tree_less,
138 .set = param_set_bool_enable_only, 170 .comp = mod_tree_comp,
139 .get = param_get_bool,
140}; 171};
141#define param_check_bool_enable_only param_check_bool
142 172
173static struct mod_tree_root {
174 struct latch_tree_root root;
175 unsigned long addr_min;
176 unsigned long addr_max;
177} mod_tree __cacheline_aligned = {
178 .addr_min = -1UL,
179};
180
181#define module_addr_min mod_tree.addr_min
182#define module_addr_max mod_tree.addr_max
183
184static noinline void __mod_tree_insert(struct mod_tree_node *node)
185{
186 latch_tree_insert(&node->node, &mod_tree.root, &mod_tree_ops);
187}
188
189static void __mod_tree_remove(struct mod_tree_node *node)
190{
191 latch_tree_erase(&node->node, &mod_tree.root, &mod_tree_ops);
192}
193
194/*
195 * These modifications: insert, remove_init and remove; are serialized by the
196 * module_mutex.
197 */
198static void mod_tree_insert(struct module *mod)
199{
200 mod->mtn_core.mod = mod;
201 mod->mtn_init.mod = mod;
202
203 __mod_tree_insert(&mod->mtn_core);
204 if (mod->init_size)
205 __mod_tree_insert(&mod->mtn_init);
206}
207
208static void mod_tree_remove_init(struct module *mod)
209{
210 if (mod->init_size)
211 __mod_tree_remove(&mod->mtn_init);
212}
213
214static void mod_tree_remove(struct module *mod)
215{
216 __mod_tree_remove(&mod->mtn_core);
217 mod_tree_remove_init(mod);
218}
219
220static struct module *mod_find(unsigned long addr)
221{
222 struct latch_tree_node *ltn;
223
224 ltn = latch_tree_find((void *)addr, &mod_tree.root, &mod_tree_ops);
225 if (!ltn)
226 return NULL;
227
228 return container_of(ltn, struct mod_tree_node, node)->mod;
229}
230
231#else /* MODULES_TREE_LOOKUP */
232
233static unsigned long module_addr_min = -1UL, module_addr_max = 0;
234
235static void mod_tree_insert(struct module *mod) { }
236static void mod_tree_remove_init(struct module *mod) { }
237static void mod_tree_remove(struct module *mod) { }
238
239static struct module *mod_find(unsigned long addr)
240{
241 struct module *mod;
242
243 list_for_each_entry_rcu(mod, &modules, list) {
244 if (within_module(addr, mod))
245 return mod;
246 }
247
248 return NULL;
249}
250
251#endif /* MODULES_TREE_LOOKUP */
252
253/*
254 * Bounds of module text, for speeding up __module_address.
255 * Protected by module_mutex.
256 */
257static void __mod_update_bounds(void *base, unsigned int size)
258{
259 unsigned long min = (unsigned long)base;
260 unsigned long max = min + size;
261
262 if (min < module_addr_min)
263 module_addr_min = min;
264 if (max > module_addr_max)
265 module_addr_max = max;
266}
267
268static void mod_update_bounds(struct module *mod)
269{
270 __mod_update_bounds(mod->module_core, mod->core_size);
271 if (mod->init_size)
272 __mod_update_bounds(mod->module_init, mod->init_size);
273}
274
275#ifdef CONFIG_KGDB_KDB
276struct list_head *kdb_modules = &modules; /* kdb needs the list of modules */
277#endif /* CONFIG_KGDB_KDB */
278
279static void module_assert_mutex(void)
280{
281 lockdep_assert_held(&module_mutex);
282}
283
284static void module_assert_mutex_or_preempt(void)
285{
286#ifdef CONFIG_LOCKDEP
287 if (unlikely(!debug_locks))
288 return;
289
290 WARN_ON(!rcu_read_lock_sched_held() &&
291 !lockdep_is_held(&module_mutex));
292#endif
293}
294
295static bool sig_enforce = IS_ENABLED(CONFIG_MODULE_SIG_FORCE);
296#ifndef CONFIG_MODULE_SIG_FORCE
143module_param(sig_enforce, bool_enable_only, 0644); 297module_param(sig_enforce, bool_enable_only, 0644);
144#endif /* !CONFIG_MODULE_SIG_FORCE */ 298#endif /* !CONFIG_MODULE_SIG_FORCE */
145#endif /* CONFIG_MODULE_SIG */
146 299
147/* Block module loading/unloading? */ 300/* Block module loading/unloading? */
148int modules_disabled = 0; 301int modules_disabled = 0;
@@ -153,10 +306,6 @@ static DECLARE_WAIT_QUEUE_HEAD(module_wq);
153 306
154static BLOCKING_NOTIFIER_HEAD(module_notify_list); 307static BLOCKING_NOTIFIER_HEAD(module_notify_list);
155 308
156/* Bounds of module allocation, for speeding __module_address.
157 * Protected by module_mutex. */
158static unsigned long module_addr_min = -1UL, module_addr_max = 0;
159
160int register_module_notifier(struct notifier_block *nb) 309int register_module_notifier(struct notifier_block *nb)
161{ 310{
162 return blocking_notifier_chain_register(&module_notify_list, nb); 311 return blocking_notifier_chain_register(&module_notify_list, nb);
@@ -318,6 +467,8 @@ bool each_symbol_section(bool (*fn)(const struct symsearch *arr,
318#endif 467#endif
319 }; 468 };
320 469
470 module_assert_mutex_or_preempt();
471
321 if (each_symbol_in_section(arr, ARRAY_SIZE(arr), NULL, fn, data)) 472 if (each_symbol_in_section(arr, ARRAY_SIZE(arr), NULL, fn, data))
322 return true; 473 return true;
323 474
@@ -457,6 +608,8 @@ static struct module *find_module_all(const char *name, size_t len,
457{ 608{
458 struct module *mod; 609 struct module *mod;
459 610
611 module_assert_mutex();
612
460 list_for_each_entry(mod, &modules, list) { 613 list_for_each_entry(mod, &modules, list) {
461 if (!even_unformed && mod->state == MODULE_STATE_UNFORMED) 614 if (!even_unformed && mod->state == MODULE_STATE_UNFORMED)
462 continue; 615 continue;
@@ -1169,11 +1322,17 @@ static inline int check_modstruct_version(Elf_Shdr *sechdrs,
1169{ 1322{
1170 const unsigned long *crc; 1323 const unsigned long *crc;
1171 1324
1172 /* Since this should be found in kernel (which can't be removed), 1325 /*
1173 * no locking is necessary. */ 1326 * Since this should be found in kernel (which can't be removed), no
1327 * locking is necessary -- use preempt_disable() to placate lockdep.
1328 */
1329 preempt_disable();
1174 if (!find_symbol(VMLINUX_SYMBOL_STR(module_layout), NULL, 1330 if (!find_symbol(VMLINUX_SYMBOL_STR(module_layout), NULL,
1175 &crc, true, false)) 1331 &crc, true, false)) {
1332 preempt_enable();
1176 BUG(); 1333 BUG();
1334 }
1335 preempt_enable();
1177 return check_version(sechdrs, versindex, 1336 return check_version(sechdrs, versindex,
1178 VMLINUX_SYMBOL_STR(module_layout), mod, crc, 1337 VMLINUX_SYMBOL_STR(module_layout), mod, crc,
1179 NULL); 1338 NULL);
@@ -1661,6 +1820,10 @@ static void mod_sysfs_fini(struct module *mod)
1661 mod_kobject_put(mod); 1820 mod_kobject_put(mod);
1662} 1821}
1663 1822
1823static void init_param_lock(struct module *mod)
1824{
1825 mutex_init(&mod->param_lock);
1826}
1664#else /* !CONFIG_SYSFS */ 1827#else /* !CONFIG_SYSFS */
1665 1828
1666static int mod_sysfs_setup(struct module *mod, 1829static int mod_sysfs_setup(struct module *mod,
@@ -1683,6 +1846,9 @@ static void del_usage_links(struct module *mod)
1683{ 1846{
1684} 1847}
1685 1848
1849static void init_param_lock(struct module *mod)
1850{
1851}
1686#endif /* CONFIG_SYSFS */ 1852#endif /* CONFIG_SYSFS */
1687 1853
1688static void mod_sysfs_teardown(struct module *mod) 1854static void mod_sysfs_teardown(struct module *mod)
@@ -1852,10 +2018,11 @@ static void free_module(struct module *mod)
1852 mutex_lock(&module_mutex); 2018 mutex_lock(&module_mutex);
1853 /* Unlink carefully: kallsyms could be walking list. */ 2019 /* Unlink carefully: kallsyms could be walking list. */
1854 list_del_rcu(&mod->list); 2020 list_del_rcu(&mod->list);
2021 mod_tree_remove(mod);
1855 /* Remove this module from bug list, this uses list_del_rcu */ 2022 /* Remove this module from bug list, this uses list_del_rcu */
1856 module_bug_cleanup(mod); 2023 module_bug_cleanup(mod);
1857 /* Wait for RCU synchronizing before releasing mod->list and buglist. */ 2024 /* Wait for RCU-sched synchronizing before releasing mod->list and buglist. */
1858 synchronize_rcu(); 2025 synchronize_sched();
1859 mutex_unlock(&module_mutex); 2026 mutex_unlock(&module_mutex);
1860 2027
1861 /* This may be NULL, but that's OK */ 2028 /* This may be NULL, but that's OK */
@@ -2384,22 +2551,6 @@ void * __weak module_alloc(unsigned long size)
2384 return vmalloc_exec(size); 2551 return vmalloc_exec(size);
2385} 2552}
2386 2553
2387static void *module_alloc_update_bounds(unsigned long size)
2388{
2389 void *ret = module_alloc(size);
2390
2391 if (ret) {
2392 mutex_lock(&module_mutex);
2393 /* Update module bounds. */
2394 if ((unsigned long)ret < module_addr_min)
2395 module_addr_min = (unsigned long)ret;
2396 if ((unsigned long)ret + size > module_addr_max)
2397 module_addr_max = (unsigned long)ret + size;
2398 mutex_unlock(&module_mutex);
2399 }
2400 return ret;
2401}
2402
2403#ifdef CONFIG_DEBUG_KMEMLEAK 2554#ifdef CONFIG_DEBUG_KMEMLEAK
2404static void kmemleak_load_module(const struct module *mod, 2555static void kmemleak_load_module(const struct module *mod,
2405 const struct load_info *info) 2556 const struct load_info *info)
@@ -2805,7 +2956,7 @@ static int move_module(struct module *mod, struct load_info *info)
2805 void *ptr; 2956 void *ptr;
2806 2957
2807 /* Do the allocs. */ 2958 /* Do the allocs. */
2808 ptr = module_alloc_update_bounds(mod->core_size); 2959 ptr = module_alloc(mod->core_size);
2809 /* 2960 /*
2810 * The pointer to this block is stored in the module structure 2961 * The pointer to this block is stored in the module structure
2811 * which is inside the block. Just mark it as not being a 2962 * which is inside the block. Just mark it as not being a
@@ -2819,7 +2970,7 @@ static int move_module(struct module *mod, struct load_info *info)
2819 mod->module_core = ptr; 2970 mod->module_core = ptr;
2820 2971
2821 if (mod->init_size) { 2972 if (mod->init_size) {
2822 ptr = module_alloc_update_bounds(mod->init_size); 2973 ptr = module_alloc(mod->init_size);
2823 /* 2974 /*
2824 * The pointer to this block is stored in the module structure 2975 * The pointer to this block is stored in the module structure
2825 * which is inside the block. This block doesn't need to be 2976 * which is inside the block. This block doesn't need to be
@@ -3119,6 +3270,7 @@ static noinline int do_init_module(struct module *mod)
3119 mod->symtab = mod->core_symtab; 3270 mod->symtab = mod->core_symtab;
3120 mod->strtab = mod->core_strtab; 3271 mod->strtab = mod->core_strtab;
3121#endif 3272#endif
3273 mod_tree_remove_init(mod);
3122 unset_module_init_ro_nx(mod); 3274 unset_module_init_ro_nx(mod);
3123 module_arch_freeing_init(mod); 3275 module_arch_freeing_init(mod);
3124 mod->module_init = NULL; 3276 mod->module_init = NULL;
@@ -3127,11 +3279,11 @@ static noinline int do_init_module(struct module *mod)
3127 mod->init_text_size = 0; 3279 mod->init_text_size = 0;
3128 /* 3280 /*
3129 * We want to free module_init, but be aware that kallsyms may be 3281 * We want to free module_init, but be aware that kallsyms may be
3130 * walking this with preempt disabled. In all the failure paths, 3282 * walking this with preempt disabled. In all the failure paths, we
3131 * we call synchronize_rcu/synchronize_sched, but we don't want 3283 * call synchronize_sched(), but we don't want to slow down the success
3132 * to slow down the success path, so use actual RCU here. 3284 * path, so use actual RCU here.
3133 */ 3285 */
3134 call_rcu(&freeinit->rcu, do_free_init); 3286 call_rcu_sched(&freeinit->rcu, do_free_init);
3135 mutex_unlock(&module_mutex); 3287 mutex_unlock(&module_mutex);
3136 wake_up_all(&module_wq); 3288 wake_up_all(&module_wq);
3137 3289
@@ -3188,7 +3340,9 @@ again:
3188 err = -EEXIST; 3340 err = -EEXIST;
3189 goto out; 3341 goto out;
3190 } 3342 }
3343 mod_update_bounds(mod);
3191 list_add_rcu(&mod->list, &modules); 3344 list_add_rcu(&mod->list, &modules);
3345 mod_tree_insert(mod);
3192 err = 0; 3346 err = 0;
3193 3347
3194out: 3348out:
@@ -3304,6 +3458,8 @@ static int load_module(struct load_info *info, const char __user *uargs,
3304 if (err) 3458 if (err)
3305 goto unlink_mod; 3459 goto unlink_mod;
3306 3460
3461 init_param_lock(mod);
3462
3307 /* Now we've got everything in the final locations, we can 3463 /* Now we've got everything in the final locations, we can
3308 * find optional sections. */ 3464 * find optional sections. */
3309 err = find_module_sections(mod, info); 3465 err = find_module_sections(mod, info);
@@ -3402,8 +3558,8 @@ static int load_module(struct load_info *info, const char __user *uargs,
3402 /* Unlink carefully: kallsyms could be walking list. */ 3558 /* Unlink carefully: kallsyms could be walking list. */
3403 list_del_rcu(&mod->list); 3559 list_del_rcu(&mod->list);
3404 wake_up_all(&module_wq); 3560 wake_up_all(&module_wq);
3405 /* Wait for RCU synchronizing before releasing mod->list. */ 3561 /* Wait for RCU-sched synchronizing before releasing mod->list. */
3406 synchronize_rcu(); 3562 synchronize_sched();
3407 mutex_unlock(&module_mutex); 3563 mutex_unlock(&module_mutex);
3408 free_module: 3564 free_module:
3409 /* Free lock-classes; relies on the preceding sync_rcu() */ 3565 /* Free lock-classes; relies on the preceding sync_rcu() */
@@ -3527,19 +3683,15 @@ const char *module_address_lookup(unsigned long addr,
3527 char **modname, 3683 char **modname,
3528 char *namebuf) 3684 char *namebuf)
3529{ 3685{
3530 struct module *mod;
3531 const char *ret = NULL; 3686 const char *ret = NULL;
3687 struct module *mod;
3532 3688
3533 preempt_disable(); 3689 preempt_disable();
3534 list_for_each_entry_rcu(mod, &modules, list) { 3690 mod = __module_address(addr);
3535 if (mod->state == MODULE_STATE_UNFORMED) 3691 if (mod) {
3536 continue; 3692 if (modname)
3537 if (within_module(addr, mod)) { 3693 *modname = mod->name;
3538 if (modname) 3694 ret = get_ksymbol(mod, addr, size, offset);
3539 *modname = mod->name;
3540 ret = get_ksymbol(mod, addr, size, offset);
3541 break;
3542 }
3543 } 3695 }
3544 /* Make a copy in here where it's safe */ 3696 /* Make a copy in here where it's safe */
3545 if (ret) { 3697 if (ret) {
@@ -3547,6 +3699,7 @@ const char *module_address_lookup(unsigned long addr,
3547 ret = namebuf; 3699 ret = namebuf;
3548 } 3700 }
3549 preempt_enable(); 3701 preempt_enable();
3702
3550 return ret; 3703 return ret;
3551} 3704}
3552 3705
@@ -3670,6 +3823,8 @@ int module_kallsyms_on_each_symbol(int (*fn)(void *, const char *,
3670 unsigned int i; 3823 unsigned int i;
3671 int ret; 3824 int ret;
3672 3825
3826 module_assert_mutex();
3827
3673 list_for_each_entry(mod, &modules, list) { 3828 list_for_each_entry(mod, &modules, list) {
3674 if (mod->state == MODULE_STATE_UNFORMED) 3829 if (mod->state == MODULE_STATE_UNFORMED)
3675 continue; 3830 continue;
@@ -3844,13 +3999,15 @@ struct module *__module_address(unsigned long addr)
3844 if (addr < module_addr_min || addr > module_addr_max) 3999 if (addr < module_addr_min || addr > module_addr_max)
3845 return NULL; 4000 return NULL;
3846 4001
3847 list_for_each_entry_rcu(mod, &modules, list) { 4002 module_assert_mutex_or_preempt();
4003
4004 mod = mod_find(addr);
4005 if (mod) {
4006 BUG_ON(!within_module(addr, mod));
3848 if (mod->state == MODULE_STATE_UNFORMED) 4007 if (mod->state == MODULE_STATE_UNFORMED)
3849 continue; 4008 mod = NULL;
3850 if (within_module(addr, mod))
3851 return mod;
3852 } 4009 }
3853 return NULL; 4010 return mod;
3854} 4011}
3855EXPORT_SYMBOL_GPL(__module_address); 4012EXPORT_SYMBOL_GPL(__module_address);
3856 4013
diff --git a/kernel/params.c b/kernel/params.c
index 30288c1e15dd..b6554aa71094 100644
--- a/kernel/params.c
+++ b/kernel/params.c
@@ -25,15 +25,34 @@
25#include <linux/slab.h> 25#include <linux/slab.h>
26#include <linux/ctype.h> 26#include <linux/ctype.h>
27 27
28/* Protects all parameters, and incidentally kmalloced_param list. */ 28#ifdef CONFIG_SYSFS
29/* Protects all built-in parameters, modules use their own param_lock */
29static DEFINE_MUTEX(param_lock); 30static DEFINE_MUTEX(param_lock);
30 31
32/* Use the module's mutex, or if built-in use the built-in mutex */
33#ifdef CONFIG_MODULES
34#define KPARAM_MUTEX(mod) ((mod) ? &(mod)->param_lock : &param_lock)
35#else
36#define KPARAM_MUTEX(mod) (&param_lock)
37#endif
38
39static inline void check_kparam_locked(struct module *mod)
40{
41 BUG_ON(!mutex_is_locked(KPARAM_MUTEX(mod)));
42}
43#else
44static inline void check_kparam_locked(struct module *mod)
45{
46}
47#endif /* !CONFIG_SYSFS */
48
31/* This just allows us to keep track of which parameters are kmalloced. */ 49/* This just allows us to keep track of which parameters are kmalloced. */
32struct kmalloced_param { 50struct kmalloced_param {
33 struct list_head list; 51 struct list_head list;
34 char val[]; 52 char val[];
35}; 53};
36static LIST_HEAD(kmalloced_params); 54static LIST_HEAD(kmalloced_params);
55static DEFINE_SPINLOCK(kmalloced_params_lock);
37 56
38static void *kmalloc_parameter(unsigned int size) 57static void *kmalloc_parameter(unsigned int size)
39{ 58{
@@ -43,7 +62,10 @@ static void *kmalloc_parameter(unsigned int size)
43 if (!p) 62 if (!p)
44 return NULL; 63 return NULL;
45 64
65 spin_lock(&kmalloced_params_lock);
46 list_add(&p->list, &kmalloced_params); 66 list_add(&p->list, &kmalloced_params);
67 spin_unlock(&kmalloced_params_lock);
68
47 return p->val; 69 return p->val;
48} 70}
49 71
@@ -52,6 +74,7 @@ static void maybe_kfree_parameter(void *param)
52{ 74{
53 struct kmalloced_param *p; 75 struct kmalloced_param *p;
54 76
77 spin_lock(&kmalloced_params_lock);
55 list_for_each_entry(p, &kmalloced_params, list) { 78 list_for_each_entry(p, &kmalloced_params, list) {
56 if (p->val == param) { 79 if (p->val == param) {
57 list_del(&p->list); 80 list_del(&p->list);
@@ -59,6 +82,7 @@ static void maybe_kfree_parameter(void *param)
59 break; 82 break;
60 } 83 }
61 } 84 }
85 spin_unlock(&kmalloced_params_lock);
62} 86}
63 87
64static char dash2underscore(char c) 88static char dash2underscore(char c)
@@ -119,10 +143,10 @@ static int parse_one(char *param,
119 return -EINVAL; 143 return -EINVAL;
120 pr_debug("handling %s with %p\n", param, 144 pr_debug("handling %s with %p\n", param,
121 params[i].ops->set); 145 params[i].ops->set);
122 mutex_lock(&param_lock); 146 kernel_param_lock(params[i].mod);
123 param_check_unsafe(&params[i]); 147 param_check_unsafe(&params[i]);
124 err = params[i].ops->set(val, &params[i]); 148 err = params[i].ops->set(val, &params[i]);
125 mutex_unlock(&param_lock); 149 kernel_param_unlock(params[i].mod);
126 return err; 150 return err;
127 } 151 }
128 } 152 }
@@ -254,7 +278,7 @@ char *parse_args(const char *doing,
254 return scnprintf(buffer, PAGE_SIZE, format, \ 278 return scnprintf(buffer, PAGE_SIZE, format, \
255 *((type *)kp->arg)); \ 279 *((type *)kp->arg)); \
256 } \ 280 } \
257 struct kernel_param_ops param_ops_##name = { \ 281 const struct kernel_param_ops param_ops_##name = { \
258 .set = param_set_##name, \ 282 .set = param_set_##name, \
259 .get = param_get_##name, \ 283 .get = param_get_##name, \
260 }; \ 284 }; \
@@ -306,7 +330,7 @@ static void param_free_charp(void *arg)
306 maybe_kfree_parameter(*((char **)arg)); 330 maybe_kfree_parameter(*((char **)arg));
307} 331}
308 332
309struct kernel_param_ops param_ops_charp = { 333const struct kernel_param_ops param_ops_charp = {
310 .set = param_set_charp, 334 .set = param_set_charp,
311 .get = param_get_charp, 335 .get = param_get_charp,
312 .free = param_free_charp, 336 .free = param_free_charp,
@@ -331,13 +355,44 @@ int param_get_bool(char *buffer, const struct kernel_param *kp)
331} 355}
332EXPORT_SYMBOL(param_get_bool); 356EXPORT_SYMBOL(param_get_bool);
333 357
334struct kernel_param_ops param_ops_bool = { 358const struct kernel_param_ops param_ops_bool = {
335 .flags = KERNEL_PARAM_OPS_FL_NOARG, 359 .flags = KERNEL_PARAM_OPS_FL_NOARG,
336 .set = param_set_bool, 360 .set = param_set_bool,
337 .get = param_get_bool, 361 .get = param_get_bool,
338}; 362};
339EXPORT_SYMBOL(param_ops_bool); 363EXPORT_SYMBOL(param_ops_bool);
340 364
365int param_set_bool_enable_only(const char *val, const struct kernel_param *kp)
366{
367 int err = 0;
368 bool new_value;
369 bool orig_value = *(bool *)kp->arg;
370 struct kernel_param dummy_kp = *kp;
371
372 dummy_kp.arg = &new_value;
373
374 err = param_set_bool(val, &dummy_kp);
375 if (err)
376 return err;
377
378 /* Don't let them unset it once it's set! */
379 if (!new_value && orig_value)
380 return -EROFS;
381
382 if (new_value)
383 err = param_set_bool(val, kp);
384
385 return err;
386}
387EXPORT_SYMBOL_GPL(param_set_bool_enable_only);
388
389const struct kernel_param_ops param_ops_bool_enable_only = {
390 .flags = KERNEL_PARAM_OPS_FL_NOARG,
391 .set = param_set_bool_enable_only,
392 .get = param_get_bool,
393};
394EXPORT_SYMBOL_GPL(param_ops_bool_enable_only);
395
341/* This one must be bool. */ 396/* This one must be bool. */
342int param_set_invbool(const char *val, const struct kernel_param *kp) 397int param_set_invbool(const char *val, const struct kernel_param *kp)
343{ 398{
@@ -359,7 +414,7 @@ int param_get_invbool(char *buffer, const struct kernel_param *kp)
359} 414}
360EXPORT_SYMBOL(param_get_invbool); 415EXPORT_SYMBOL(param_get_invbool);
361 416
362struct kernel_param_ops param_ops_invbool = { 417const struct kernel_param_ops param_ops_invbool = {
363 .set = param_set_invbool, 418 .set = param_set_invbool,
364 .get = param_get_invbool, 419 .get = param_get_invbool,
365}; 420};
@@ -367,12 +422,11 @@ EXPORT_SYMBOL(param_ops_invbool);
367 422
368int param_set_bint(const char *val, const struct kernel_param *kp) 423int param_set_bint(const char *val, const struct kernel_param *kp)
369{ 424{
370 struct kernel_param boolkp; 425 /* Match bool exactly, by re-using it. */
426 struct kernel_param boolkp = *kp;
371 bool v; 427 bool v;
372 int ret; 428 int ret;
373 429
374 /* Match bool exactly, by re-using it. */
375 boolkp = *kp;
376 boolkp.arg = &v; 430 boolkp.arg = &v;
377 431
378 ret = param_set_bool(val, &boolkp); 432 ret = param_set_bool(val, &boolkp);
@@ -382,7 +436,7 @@ int param_set_bint(const char *val, const struct kernel_param *kp)
382} 436}
383EXPORT_SYMBOL(param_set_bint); 437EXPORT_SYMBOL(param_set_bint);
384 438
385struct kernel_param_ops param_ops_bint = { 439const struct kernel_param_ops param_ops_bint = {
386 .flags = KERNEL_PARAM_OPS_FL_NOARG, 440 .flags = KERNEL_PARAM_OPS_FL_NOARG,
387 .set = param_set_bint, 441 .set = param_set_bint,
388 .get = param_get_int, 442 .get = param_get_int,
@@ -390,7 +444,8 @@ struct kernel_param_ops param_ops_bint = {
390EXPORT_SYMBOL(param_ops_bint); 444EXPORT_SYMBOL(param_ops_bint);
391 445
392/* We break the rule and mangle the string. */ 446/* We break the rule and mangle the string. */
393static int param_array(const char *name, 447static int param_array(struct module *mod,
448 const char *name,
394 const char *val, 449 const char *val,
395 unsigned int min, unsigned int max, 450 unsigned int min, unsigned int max,
396 void *elem, int elemsize, 451 void *elem, int elemsize,
@@ -421,7 +476,7 @@ static int param_array(const char *name,
421 /* nul-terminate and parse */ 476 /* nul-terminate and parse */
422 save = val[len]; 477 save = val[len];
423 ((char *)val)[len] = '\0'; 478 ((char *)val)[len] = '\0';
424 BUG_ON(!mutex_is_locked(&param_lock)); 479 check_kparam_locked(mod);
425 ret = set(val, &kp); 480 ret = set(val, &kp);
426 481
427 if (ret != 0) 482 if (ret != 0)
@@ -443,7 +498,7 @@ static int param_array_set(const char *val, const struct kernel_param *kp)
443 const struct kparam_array *arr = kp->arr; 498 const struct kparam_array *arr = kp->arr;
444 unsigned int temp_num; 499 unsigned int temp_num;
445 500
446 return param_array(kp->name, val, 1, arr->max, arr->elem, 501 return param_array(kp->mod, kp->name, val, 1, arr->max, arr->elem,
447 arr->elemsize, arr->ops->set, kp->level, 502 arr->elemsize, arr->ops->set, kp->level,
448 arr->num ?: &temp_num); 503 arr->num ?: &temp_num);
449} 504}
@@ -452,14 +507,13 @@ static int param_array_get(char *buffer, const struct kernel_param *kp)
452{ 507{
453 int i, off, ret; 508 int i, off, ret;
454 const struct kparam_array *arr = kp->arr; 509 const struct kparam_array *arr = kp->arr;
455 struct kernel_param p; 510 struct kernel_param p = *kp;
456 511
457 p = *kp;
458 for (i = off = 0; i < (arr->num ? *arr->num : arr->max); i++) { 512 for (i = off = 0; i < (arr->num ? *arr->num : arr->max); i++) {
459 if (i) 513 if (i)
460 buffer[off++] = ','; 514 buffer[off++] = ',';
461 p.arg = arr->elem + arr->elemsize * i; 515 p.arg = arr->elem + arr->elemsize * i;
462 BUG_ON(!mutex_is_locked(&param_lock)); 516 check_kparam_locked(p.mod);
463 ret = arr->ops->get(buffer + off, &p); 517 ret = arr->ops->get(buffer + off, &p);
464 if (ret < 0) 518 if (ret < 0)
465 return ret; 519 return ret;
@@ -479,7 +533,7 @@ static void param_array_free(void *arg)
479 arr->ops->free(arr->elem + arr->elemsize * i); 533 arr->ops->free(arr->elem + arr->elemsize * i);
480} 534}
481 535
482struct kernel_param_ops param_array_ops = { 536const struct kernel_param_ops param_array_ops = {
483 .set = param_array_set, 537 .set = param_array_set,
484 .get = param_array_get, 538 .get = param_array_get,
485 .free = param_array_free, 539 .free = param_array_free,
@@ -507,7 +561,7 @@ int param_get_string(char *buffer, const struct kernel_param *kp)
507} 561}
508EXPORT_SYMBOL(param_get_string); 562EXPORT_SYMBOL(param_get_string);
509 563
510struct kernel_param_ops param_ops_string = { 564const struct kernel_param_ops param_ops_string = {
511 .set = param_set_copystring, 565 .set = param_set_copystring,
512 .get = param_get_string, 566 .get = param_get_string,
513}; 567};
@@ -542,9 +596,9 @@ static ssize_t param_attr_show(struct module_attribute *mattr,
542 if (!attribute->param->ops->get) 596 if (!attribute->param->ops->get)
543 return -EPERM; 597 return -EPERM;
544 598
545 mutex_lock(&param_lock); 599 kernel_param_lock(mk->mod);
546 count = attribute->param->ops->get(buf, attribute->param); 600 count = attribute->param->ops->get(buf, attribute->param);
547 mutex_unlock(&param_lock); 601 kernel_param_unlock(mk->mod);
548 if (count > 0) { 602 if (count > 0) {
549 strcat(buf, "\n"); 603 strcat(buf, "\n");
550 ++count; 604 ++count;
@@ -554,7 +608,7 @@ static ssize_t param_attr_show(struct module_attribute *mattr,
554 608
555/* sysfs always hands a nul-terminated string in buf. We rely on that. */ 609/* sysfs always hands a nul-terminated string in buf. We rely on that. */
556static ssize_t param_attr_store(struct module_attribute *mattr, 610static ssize_t param_attr_store(struct module_attribute *mattr,
557 struct module_kobject *km, 611 struct module_kobject *mk,
558 const char *buf, size_t len) 612 const char *buf, size_t len)
559{ 613{
560 int err; 614 int err;
@@ -563,10 +617,10 @@ static ssize_t param_attr_store(struct module_attribute *mattr,
563 if (!attribute->param->ops->set) 617 if (!attribute->param->ops->set)
564 return -EPERM; 618 return -EPERM;
565 619
566 mutex_lock(&param_lock); 620 kernel_param_lock(mk->mod);
567 param_check_unsafe(attribute->param); 621 param_check_unsafe(attribute->param);
568 err = attribute->param->ops->set(buf, attribute->param); 622 err = attribute->param->ops->set(buf, attribute->param);
569 mutex_unlock(&param_lock); 623 kernel_param_unlock(mk->mod);
570 if (!err) 624 if (!err)
571 return len; 625 return len;
572 return err; 626 return err;
@@ -580,17 +634,18 @@ static ssize_t param_attr_store(struct module_attribute *mattr,
580#endif 634#endif
581 635
582#ifdef CONFIG_SYSFS 636#ifdef CONFIG_SYSFS
583void __kernel_param_lock(void) 637void kernel_param_lock(struct module *mod)
584{ 638{
585 mutex_lock(&param_lock); 639 mutex_lock(KPARAM_MUTEX(mod));
586} 640}
587EXPORT_SYMBOL(__kernel_param_lock);
588 641
589void __kernel_param_unlock(void) 642void kernel_param_unlock(struct module *mod)
590{ 643{
591 mutex_unlock(&param_lock); 644 mutex_unlock(KPARAM_MUTEX(mod));
592} 645}
593EXPORT_SYMBOL(__kernel_param_unlock); 646
647EXPORT_SYMBOL(kernel_param_lock);
648EXPORT_SYMBOL(kernel_param_unlock);
594 649
595/* 650/*
596 * add_sysfs_param - add a parameter to sysfs 651 * add_sysfs_param - add a parameter to sysfs
@@ -856,6 +911,7 @@ static void __init version_sysfs_builtin(void)
856 mk = locate_module_kobject(vattr->module_name); 911 mk = locate_module_kobject(vattr->module_name);
857 if (mk) { 912 if (mk) {
858 err = sysfs_create_file(&mk->kobj, &vattr->mattr.attr); 913 err = sysfs_create_file(&mk->kobj, &vattr->mattr.attr);
914 WARN_ON_ONCE(err);
859 kobject_uevent(&mk->kobj, KOBJ_ADD); 915 kobject_uevent(&mk->kobj, KOBJ_ADD);
860 kobject_put(&mk->kobj); 916 kobject_put(&mk->kobj);
861 } 917 }
diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c
index 30b7a409bf1e..bca3667a2de1 100644
--- a/kernel/time/timekeeping.c
+++ b/kernel/time/timekeeping.c
@@ -319,32 +319,7 @@ static inline s64 timekeeping_get_ns(struct tk_read_base *tkr)
319 * We want to use this from any context including NMI and tracing / 319 * We want to use this from any context including NMI and tracing /
320 * instrumenting the timekeeping code itself. 320 * instrumenting the timekeeping code itself.
321 * 321 *
322 * So we handle this differently than the other timekeeping accessor 322 * Employ the latch technique; see @raw_write_seqcount_latch.
323 * functions which retry when the sequence count has changed. The
324 * update side does:
325 *
326 * smp_wmb(); <- Ensure that the last base[1] update is visible
327 * tkf->seq++;
328 * smp_wmb(); <- Ensure that the seqcount update is visible
329 * update(tkf->base[0], tkr);
330 * smp_wmb(); <- Ensure that the base[0] update is visible
331 * tkf->seq++;
332 * smp_wmb(); <- Ensure that the seqcount update is visible
333 * update(tkf->base[1], tkr);
334 *
335 * The reader side does:
336 *
337 * do {
338 * seq = tkf->seq;
339 * smp_rmb();
340 * idx = seq & 0x01;
341 * now = now(tkf->base[idx]);
342 * smp_rmb();
343 * } while (seq != tkf->seq)
344 *
345 * As long as we update base[0] readers are forced off to
346 * base[1]. Once base[0] is updated readers are redirected to base[0]
347 * and the base[1] update takes place.
348 * 323 *
349 * So if a NMI hits the update of base[0] then it will use base[1] 324 * So if a NMI hits the update of base[0] then it will use base[1]
350 * which is still consistent. In the worst case this can result is a 325 * which is still consistent. In the worst case this can result is a
@@ -407,7 +382,7 @@ static __always_inline u64 __ktime_get_fast_ns(struct tk_fast *tkf)
407 u64 now; 382 u64 now;
408 383
409 do { 384 do {
410 seq = raw_read_seqcount(&tkf->seq); 385 seq = raw_read_seqcount_latch(&tkf->seq);
411 tkr = tkf->base + (seq & 0x01); 386 tkr = tkf->base + (seq & 0x01);
412 now = ktime_to_ns(tkr->base) + timekeeping_get_ns(tkr); 387 now = ktime_to_ns(tkr->base) + timekeeping_get_ns(tkr);
413 } while (read_seqcount_retry(&tkf->seq, seq)); 388 } while (read_seqcount_retry(&tkf->seq, seq));
diff --git a/kernel/workqueue.c b/kernel/workqueue.c
index 5243d4b03087..4c4f06176f74 100644
--- a/kernel/workqueue.c
+++ b/kernel/workqueue.c
@@ -285,12 +285,7 @@ static bool wq_disable_numa;
285module_param_named(disable_numa, wq_disable_numa, bool, 0444); 285module_param_named(disable_numa, wq_disable_numa, bool, 0444);
286 286
287/* see the comment above the definition of WQ_POWER_EFFICIENT */ 287/* see the comment above the definition of WQ_POWER_EFFICIENT */
288#ifdef CONFIG_WQ_POWER_EFFICIENT_DEFAULT 288static bool wq_power_efficient = IS_ENABLED(CONFIG_WQ_POWER_EFFICIENT_DEFAULT);
289static bool wq_power_efficient = true;
290#else
291static bool wq_power_efficient;
292#endif
293
294module_param_named(power_efficient, wq_power_efficient, bool, 0444); 289module_param_named(power_efficient, wq_power_efficient, bool, 0444);
295 290
296static bool wq_numa_enabled; /* unbound NUMA affinity enabled */ 291static bool wq_numa_enabled; /* unbound NUMA affinity enabled */
diff --git a/lib/bug.c b/lib/bug.c
index 0c3bd9552b6f..cff145f032a5 100644
--- a/lib/bug.c
+++ b/lib/bug.c
@@ -66,7 +66,7 @@ static const struct bug_entry *module_find_bug(unsigned long bugaddr)
66 struct module *mod; 66 struct module *mod;
67 const struct bug_entry *bug = NULL; 67 const struct bug_entry *bug = NULL;
68 68
69 rcu_read_lock(); 69 rcu_read_lock_sched();
70 list_for_each_entry_rcu(mod, &module_bug_list, bug_list) { 70 list_for_each_entry_rcu(mod, &module_bug_list, bug_list) {
71 unsigned i; 71 unsigned i;
72 72
@@ -77,7 +77,7 @@ static const struct bug_entry *module_find_bug(unsigned long bugaddr)
77 } 77 }
78 bug = NULL; 78 bug = NULL;
79out: 79out:
80 rcu_read_unlock(); 80 rcu_read_unlock_sched();
81 81
82 return bug; 82 return bug;
83} 83}
@@ -88,6 +88,8 @@ void module_bug_finalize(const Elf_Ehdr *hdr, const Elf_Shdr *sechdrs,
88 char *secstrings; 88 char *secstrings;
89 unsigned int i; 89 unsigned int i;
90 90
91 lockdep_assert_held(&module_mutex);
92
91 mod->bug_table = NULL; 93 mod->bug_table = NULL;
92 mod->num_bugs = 0; 94 mod->num_bugs = 0;
93 95
@@ -113,6 +115,7 @@ void module_bug_finalize(const Elf_Ehdr *hdr, const Elf_Shdr *sechdrs,
113 115
114void module_bug_cleanup(struct module *mod) 116void module_bug_cleanup(struct module *mod)
115{ 117{
118 lockdep_assert_held(&module_mutex);
116 list_del_rcu(&mod->bug_list); 119 list_del_rcu(&mod->bug_list);
117} 120}
118 121
diff --git a/lib/rbtree.c b/lib/rbtree.c
index c16c81a3d430..1356454e36de 100644
--- a/lib/rbtree.c
+++ b/lib/rbtree.c
@@ -44,6 +44,30 @@
44 * parentheses and have some accompanying text comment. 44 * parentheses and have some accompanying text comment.
45 */ 45 */
46 46
47/*
48 * Notes on lockless lookups:
49 *
50 * All stores to the tree structure (rb_left and rb_right) must be done using
51 * WRITE_ONCE(). And we must not inadvertently cause (temporary) loops in the
52 * tree structure as seen in program order.
53 *
54 * These two requirements will allow lockless iteration of the tree -- not
55 * correct iteration mind you, tree rotations are not atomic so a lookup might
56 * miss entire subtrees.
57 *
58 * But they do guarantee that any such traversal will only see valid elements
59 * and that it will indeed complete -- does not get stuck in a loop.
60 *
61 * It also guarantees that if the lookup returns an element it is the 'correct'
62 * one. But not returning an element does _NOT_ mean it's not present.
63 *
64 * NOTE:
65 *
66 * Stores to __rb_parent_color are not important for simple lookups so those
67 * are left undone as of now. Nor did I check for loops involving parent
68 * pointers.
69 */
70
47static inline void rb_set_black(struct rb_node *rb) 71static inline void rb_set_black(struct rb_node *rb)
48{ 72{
49 rb->__rb_parent_color |= RB_BLACK; 73 rb->__rb_parent_color |= RB_BLACK;
@@ -129,8 +153,9 @@ __rb_insert(struct rb_node *node, struct rb_root *root,
129 * This still leaves us in violation of 4), the 153 * This still leaves us in violation of 4), the
130 * continuation into Case 3 will fix that. 154 * continuation into Case 3 will fix that.
131 */ 155 */
132 parent->rb_right = tmp = node->rb_left; 156 tmp = node->rb_left;
133 node->rb_left = parent; 157 WRITE_ONCE(parent->rb_right, tmp);
158 WRITE_ONCE(node->rb_left, parent);
134 if (tmp) 159 if (tmp)
135 rb_set_parent_color(tmp, parent, 160 rb_set_parent_color(tmp, parent,
136 RB_BLACK); 161 RB_BLACK);
@@ -149,8 +174,8 @@ __rb_insert(struct rb_node *node, struct rb_root *root,
149 * / \ 174 * / \
150 * n U 175 * n U
151 */ 176 */
152 gparent->rb_left = tmp; /* == parent->rb_right */ 177 WRITE_ONCE(gparent->rb_left, tmp); /* == parent->rb_right */
153 parent->rb_right = gparent; 178 WRITE_ONCE(parent->rb_right, gparent);
154 if (tmp) 179 if (tmp)
155 rb_set_parent_color(tmp, gparent, RB_BLACK); 180 rb_set_parent_color(tmp, gparent, RB_BLACK);
156 __rb_rotate_set_parents(gparent, parent, root, RB_RED); 181 __rb_rotate_set_parents(gparent, parent, root, RB_RED);
@@ -171,8 +196,9 @@ __rb_insert(struct rb_node *node, struct rb_root *root,
171 tmp = parent->rb_left; 196 tmp = parent->rb_left;
172 if (node == tmp) { 197 if (node == tmp) {
173 /* Case 2 - right rotate at parent */ 198 /* Case 2 - right rotate at parent */
174 parent->rb_left = tmp = node->rb_right; 199 tmp = node->rb_right;
175 node->rb_right = parent; 200 WRITE_ONCE(parent->rb_left, tmp);
201 WRITE_ONCE(node->rb_right, parent);
176 if (tmp) 202 if (tmp)
177 rb_set_parent_color(tmp, parent, 203 rb_set_parent_color(tmp, parent,
178 RB_BLACK); 204 RB_BLACK);
@@ -183,8 +209,8 @@ __rb_insert(struct rb_node *node, struct rb_root *root,
183 } 209 }
184 210
185 /* Case 3 - left rotate at gparent */ 211 /* Case 3 - left rotate at gparent */
186 gparent->rb_right = tmp; /* == parent->rb_left */ 212 WRITE_ONCE(gparent->rb_right, tmp); /* == parent->rb_left */
187 parent->rb_left = gparent; 213 WRITE_ONCE(parent->rb_left, gparent);
188 if (tmp) 214 if (tmp)
189 rb_set_parent_color(tmp, gparent, RB_BLACK); 215 rb_set_parent_color(tmp, gparent, RB_BLACK);
190 __rb_rotate_set_parents(gparent, parent, root, RB_RED); 216 __rb_rotate_set_parents(gparent, parent, root, RB_RED);
@@ -224,8 +250,9 @@ ____rb_erase_color(struct rb_node *parent, struct rb_root *root,
224 * / \ / \ 250 * / \ / \
225 * Sl Sr N Sl 251 * Sl Sr N Sl
226 */ 252 */
227 parent->rb_right = tmp1 = sibling->rb_left; 253 tmp1 = sibling->rb_left;
228 sibling->rb_left = parent; 254 WRITE_ONCE(parent->rb_right, tmp1);
255 WRITE_ONCE(sibling->rb_left, parent);
229 rb_set_parent_color(tmp1, parent, RB_BLACK); 256 rb_set_parent_color(tmp1, parent, RB_BLACK);
230 __rb_rotate_set_parents(parent, sibling, root, 257 __rb_rotate_set_parents(parent, sibling, root,
231 RB_RED); 258 RB_RED);
@@ -275,9 +302,10 @@ ____rb_erase_color(struct rb_node *parent, struct rb_root *root,
275 * \ 302 * \
276 * Sr 303 * Sr
277 */ 304 */
278 sibling->rb_left = tmp1 = tmp2->rb_right; 305 tmp1 = tmp2->rb_right;
279 tmp2->rb_right = sibling; 306 WRITE_ONCE(sibling->rb_left, tmp1);
280 parent->rb_right = tmp2; 307 WRITE_ONCE(tmp2->rb_right, sibling);
308 WRITE_ONCE(parent->rb_right, tmp2);
281 if (tmp1) 309 if (tmp1)
282 rb_set_parent_color(tmp1, sibling, 310 rb_set_parent_color(tmp1, sibling,
283 RB_BLACK); 311 RB_BLACK);
@@ -297,8 +325,9 @@ ____rb_erase_color(struct rb_node *parent, struct rb_root *root,
297 * / \ / \ 325 * / \ / \
298 * (sl) sr N (sl) 326 * (sl) sr N (sl)
299 */ 327 */
300 parent->rb_right = tmp2 = sibling->rb_left; 328 tmp2 = sibling->rb_left;
301 sibling->rb_left = parent; 329 WRITE_ONCE(parent->rb_right, tmp2);
330 WRITE_ONCE(sibling->rb_left, parent);
302 rb_set_parent_color(tmp1, sibling, RB_BLACK); 331 rb_set_parent_color(tmp1, sibling, RB_BLACK);
303 if (tmp2) 332 if (tmp2)
304 rb_set_parent(tmp2, parent); 333 rb_set_parent(tmp2, parent);
@@ -310,8 +339,9 @@ ____rb_erase_color(struct rb_node *parent, struct rb_root *root,
310 sibling = parent->rb_left; 339 sibling = parent->rb_left;
311 if (rb_is_red(sibling)) { 340 if (rb_is_red(sibling)) {
312 /* Case 1 - right rotate at parent */ 341 /* Case 1 - right rotate at parent */
313 parent->rb_left = tmp1 = sibling->rb_right; 342 tmp1 = sibling->rb_right;
314 sibling->rb_right = parent; 343 WRITE_ONCE(parent->rb_left, tmp1);
344 WRITE_ONCE(sibling->rb_right, parent);
315 rb_set_parent_color(tmp1, parent, RB_BLACK); 345 rb_set_parent_color(tmp1, parent, RB_BLACK);
316 __rb_rotate_set_parents(parent, sibling, root, 346 __rb_rotate_set_parents(parent, sibling, root,
317 RB_RED); 347 RB_RED);
@@ -336,9 +366,10 @@ ____rb_erase_color(struct rb_node *parent, struct rb_root *root,
336 break; 366 break;
337 } 367 }
338 /* Case 3 - right rotate at sibling */ 368 /* Case 3 - right rotate at sibling */
339 sibling->rb_right = tmp1 = tmp2->rb_left; 369 tmp1 = tmp2->rb_left;
340 tmp2->rb_left = sibling; 370 WRITE_ONCE(sibling->rb_right, tmp1);
341 parent->rb_left = tmp2; 371 WRITE_ONCE(tmp2->rb_left, sibling);
372 WRITE_ONCE(parent->rb_left, tmp2);
342 if (tmp1) 373 if (tmp1)
343 rb_set_parent_color(tmp1, sibling, 374 rb_set_parent_color(tmp1, sibling,
344 RB_BLACK); 375 RB_BLACK);
@@ -347,8 +378,9 @@ ____rb_erase_color(struct rb_node *parent, struct rb_root *root,
347 sibling = tmp2; 378 sibling = tmp2;
348 } 379 }
349 /* Case 4 - left rotate at parent + color flips */ 380 /* Case 4 - left rotate at parent + color flips */
350 parent->rb_left = tmp2 = sibling->rb_right; 381 tmp2 = sibling->rb_right;
351 sibling->rb_right = parent; 382 WRITE_ONCE(parent->rb_left, tmp2);
383 WRITE_ONCE(sibling->rb_right, parent);
352 rb_set_parent_color(tmp1, sibling, RB_BLACK); 384 rb_set_parent_color(tmp1, sibling, RB_BLACK);
353 if (tmp2) 385 if (tmp2)
354 rb_set_parent(tmp2, parent); 386 rb_set_parent(tmp2, parent);
diff --git a/net/mac80211/rate.c b/net/mac80211/rate.c
index 36ba7c4f0283..fda33f961d83 100644
--- a/net/mac80211/rate.c
+++ b/net/mac80211/rate.c
@@ -103,7 +103,7 @@ ieee80211_rate_control_ops_get(const char *name)
103 const struct rate_control_ops *ops; 103 const struct rate_control_ops *ops;
104 const char *alg_name; 104 const char *alg_name;
105 105
106 kparam_block_sysfs_write(ieee80211_default_rc_algo); 106 kernel_param_lock(THIS_MODULE);
107 if (!name) 107 if (!name)
108 alg_name = ieee80211_default_rc_algo; 108 alg_name = ieee80211_default_rc_algo;
109 else 109 else
@@ -117,7 +117,7 @@ ieee80211_rate_control_ops_get(const char *name)
117 /* try built-in one if specific alg requested but not found */ 117 /* try built-in one if specific alg requested but not found */
118 if (!ops && strlen(CONFIG_MAC80211_RC_DEFAULT)) 118 if (!ops && strlen(CONFIG_MAC80211_RC_DEFAULT))
119 ops = ieee80211_try_rate_control_ops_get(CONFIG_MAC80211_RC_DEFAULT); 119 ops = ieee80211_try_rate_control_ops_get(CONFIG_MAC80211_RC_DEFAULT);
120 kparam_unblock_sysfs_write(ieee80211_default_rc_algo); 120 kernel_param_unlock(THIS_MODULE);
121 121
122 return ops; 122 return ops;
123} 123}
diff --git a/net/sunrpc/auth.c b/net/sunrpc/auth.c
index 47f38be4155f..02f53674dc39 100644
--- a/net/sunrpc/auth.c
+++ b/net/sunrpc/auth.c
@@ -72,7 +72,7 @@ static int param_get_hashtbl_sz(char *buffer, const struct kernel_param *kp)
72 72
73#define param_check_hashtbl_sz(name, p) __param_check(name, p, unsigned int); 73#define param_check_hashtbl_sz(name, p) __param_check(name, p, unsigned int);
74 74
75static struct kernel_param_ops param_ops_hashtbl_sz = { 75static const struct kernel_param_ops param_ops_hashtbl_sz = {
76 .set = param_set_hashtbl_sz, 76 .set = param_set_hashtbl_sz,
77 .get = param_get_hashtbl_sz, 77 .get = param_get_hashtbl_sz,
78}; 78};
diff --git a/net/sunrpc/xprtsock.c b/net/sunrpc/xprtsock.c
index 66891e32c5e3..b0517287075b 100644
--- a/net/sunrpc/xprtsock.c
+++ b/net/sunrpc/xprtsock.c
@@ -2982,7 +2982,7 @@ static int param_set_portnr(const char *val, const struct kernel_param *kp)
2982 RPC_MAX_RESVPORT); 2982 RPC_MAX_RESVPORT);
2983} 2983}
2984 2984
2985static struct kernel_param_ops param_ops_portnr = { 2985static const struct kernel_param_ops param_ops_portnr = {
2986 .set = param_set_portnr, 2986 .set = param_set_portnr,
2987 .get = param_get_uint, 2987 .get = param_get_uint,
2988}; 2988};
@@ -3001,7 +3001,7 @@ static int param_set_slot_table_size(const char *val,
3001 RPC_MAX_SLOT_TABLE); 3001 RPC_MAX_SLOT_TABLE);
3002} 3002}
3003 3003
3004static struct kernel_param_ops param_ops_slot_table_size = { 3004static const struct kernel_param_ops param_ops_slot_table_size = {
3005 .set = param_set_slot_table_size, 3005 .set = param_set_slot_table_size,
3006 .get = param_get_uint, 3006 .get = param_get_uint,
3007}; 3007};
@@ -3017,7 +3017,7 @@ static int param_set_max_slot_table_size(const char *val,
3017 RPC_MAX_SLOT_TABLE_LIMIT); 3017 RPC_MAX_SLOT_TABLE_LIMIT);
3018} 3018}
3019 3019
3020static struct kernel_param_ops param_ops_max_slot_table_size = { 3020static const struct kernel_param_ops param_ops_max_slot_table_size = {
3021 .set = param_set_max_slot_table_size, 3021 .set = param_set_max_slot_table_size,
3022 .get = param_get_uint, 3022 .get = param_get_uint,
3023}; 3023};
diff --git a/security/apparmor/lsm.c b/security/apparmor/lsm.c
index 5696874e8062..dec607c17b64 100644
--- a/security/apparmor/lsm.c
+++ b/security/apparmor/lsm.c
@@ -654,7 +654,7 @@ static struct security_hook_list apparmor_hooks[] = {
654static int param_set_aabool(const char *val, const struct kernel_param *kp); 654static int param_set_aabool(const char *val, const struct kernel_param *kp);
655static int param_get_aabool(char *buffer, const struct kernel_param *kp); 655static int param_get_aabool(char *buffer, const struct kernel_param *kp);
656#define param_check_aabool param_check_bool 656#define param_check_aabool param_check_bool
657static struct kernel_param_ops param_ops_aabool = { 657static const struct kernel_param_ops param_ops_aabool = {
658 .flags = KERNEL_PARAM_OPS_FL_NOARG, 658 .flags = KERNEL_PARAM_OPS_FL_NOARG,
659 .set = param_set_aabool, 659 .set = param_set_aabool,
660 .get = param_get_aabool 660 .get = param_get_aabool
@@ -663,7 +663,7 @@ static struct kernel_param_ops param_ops_aabool = {
663static int param_set_aauint(const char *val, const struct kernel_param *kp); 663static int param_set_aauint(const char *val, const struct kernel_param *kp);
664static int param_get_aauint(char *buffer, const struct kernel_param *kp); 664static int param_get_aauint(char *buffer, const struct kernel_param *kp);
665#define param_check_aauint param_check_uint 665#define param_check_aauint param_check_uint
666static struct kernel_param_ops param_ops_aauint = { 666static const struct kernel_param_ops param_ops_aauint = {
667 .set = param_set_aauint, 667 .set = param_set_aauint,
668 .get = param_get_aauint 668 .get = param_get_aauint
669}; 669};
@@ -671,7 +671,7 @@ static struct kernel_param_ops param_ops_aauint = {
671static int param_set_aalockpolicy(const char *val, const struct kernel_param *kp); 671static int param_set_aalockpolicy(const char *val, const struct kernel_param *kp);
672static int param_get_aalockpolicy(char *buffer, const struct kernel_param *kp); 672static int param_get_aalockpolicy(char *buffer, const struct kernel_param *kp);
673#define param_check_aalockpolicy param_check_bool 673#define param_check_aalockpolicy param_check_bool
674static struct kernel_param_ops param_ops_aalockpolicy = { 674static const struct kernel_param_ops param_ops_aalockpolicy = {
675 .flags = KERNEL_PARAM_OPS_FL_NOARG, 675 .flags = KERNEL_PARAM_OPS_FL_NOARG,
676 .set = param_set_aalockpolicy, 676 .set = param_set_aalockpolicy,
677 .get = param_get_aalockpolicy 677 .get = param_get_aalockpolicy
diff --git a/security/integrity/ima/ima_crypto.c b/security/integrity/ima/ima_crypto.c
index 686355fea7fd..e24121afb2f2 100644
--- a/security/integrity/ima/ima_crypto.c
+++ b/security/integrity/ima/ima_crypto.c
@@ -55,7 +55,7 @@ static int param_set_bufsize(const char *val, const struct kernel_param *kp)
55 return 0; 55 return 0;
56} 56}
57 57
58static struct kernel_param_ops param_ops_bufsize = { 58static const struct kernel_param_ops param_ops_bufsize = {
59 .set = param_set_bufsize, 59 .set = param_set_bufsize,
60 .get = param_get_uint, 60 .get = param_get_uint,
61}; 61};
diff --git a/sound/pci/hda/hda_intel.c b/sound/pci/hda/hda_intel.c
index 7dea7987d2af..9913e24d6699 100644
--- a/sound/pci/hda/hda_intel.c
+++ b/sound/pci/hda/hda_intel.c
@@ -171,7 +171,7 @@ MODULE_PARM_DESC(beep_mode, "Select HDA Beep registration mode "
171 171
172#ifdef CONFIG_PM 172#ifdef CONFIG_PM
173static int param_set_xint(const char *val, const struct kernel_param *kp); 173static int param_set_xint(const char *val, const struct kernel_param *kp);
174static struct kernel_param_ops param_ops_xint = { 174static const struct kernel_param_ops param_ops_xint = {
175 .set = param_set_xint, 175 .set = param_set_xint,
176 .get = param_get_int, 176 .get = param_get_int,
177}; 177};
generated by cgit v1.2.2 (git 2.25.0) at 2025-04-15 05:30:59 -0400