aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--Documentation/lockstat.txt1
-rw-r--r--arch/x86/lib/usercopy_32.c8
-rw-r--r--arch/x86/lib/usercopy_64.c4
-rw-r--r--include/asm-um/system-generic.h32
-rw-r--r--include/asm-x86/uaccess.h2
-rw-r--r--include/asm-x86/uaccess_32.h8
-rw-r--r--include/asm-x86/uaccess_64.h6
-rw-r--r--include/linux/debug_locks.h2
-rw-r--r--include/linux/kernel.h9
-rw-r--r--include/linux/lockdep.h18
-rw-r--r--include/linux/uaccess.h2
-rw-r--r--mm/memory.c15
12 files changed, 79 insertions, 28 deletions
diff --git a/Documentation/lockstat.txt b/Documentation/lockstat.txt
index 4ba4664ce5c3..02f36f5c64fe 100644
--- a/Documentation/lockstat.txt
+++ b/Documentation/lockstat.txt
@@ -100,6 +100,7 @@ The first lock (05-10) is a read/write lock, and shows two lines above the
100short separator. The contention points don't match the column descriptors, 100short separator. The contention points don't match the column descriptors,
101they have two: contentions and [<IP>] symbol. 101they have two: contentions and [<IP>] symbol.
102 102
103The integer part of the time values is in us.
103 104
104View the top contending locks: 105View the top contending locks:
105 106
diff --git a/arch/x86/lib/usercopy_32.c b/arch/x86/lib/usercopy_32.c
index 24e60944971a..fab5faba1d3e 100644
--- a/arch/x86/lib/usercopy_32.c
+++ b/arch/x86/lib/usercopy_32.c
@@ -32,7 +32,7 @@ static inline int __movsl_is_ok(unsigned long a1, unsigned long a2, unsigned lon
32#define __do_strncpy_from_user(dst, src, count, res) \ 32#define __do_strncpy_from_user(dst, src, count, res) \
33do { \ 33do { \
34 int __d0, __d1, __d2; \ 34 int __d0, __d1, __d2; \
35 might_sleep(); \ 35 might_fault(); \
36 __asm__ __volatile__( \ 36 __asm__ __volatile__( \
37 " testl %1,%1\n" \ 37 " testl %1,%1\n" \
38 " jz 2f\n" \ 38 " jz 2f\n" \
@@ -119,7 +119,7 @@ EXPORT_SYMBOL(strncpy_from_user);
119#define __do_clear_user(addr,size) \ 119#define __do_clear_user(addr,size) \
120do { \ 120do { \
121 int __d0; \ 121 int __d0; \
122 might_sleep(); \ 122 might_fault(); \
123 __asm__ __volatile__( \ 123 __asm__ __volatile__( \
124 "0: rep; stosl\n" \ 124 "0: rep; stosl\n" \
125 " movl %2,%0\n" \ 125 " movl %2,%0\n" \
@@ -148,7 +148,7 @@ do { \
148unsigned long 148unsigned long
149clear_user(void __user *to, unsigned long n) 149clear_user(void __user *to, unsigned long n)
150{ 150{
151 might_sleep(); 151 might_fault();
152 if (access_ok(VERIFY_WRITE, to, n)) 152 if (access_ok(VERIFY_WRITE, to, n))
153 __do_clear_user(to, n); 153 __do_clear_user(to, n);
154 return n; 154 return n;
@@ -190,7 +190,7 @@ long strnlen_user(const char __user *s, long n)
190 unsigned long mask = -__addr_ok(s); 190 unsigned long mask = -__addr_ok(s);
191 unsigned long res, tmp; 191 unsigned long res, tmp;
192 192
193 might_sleep(); 193 might_fault();
194 194
195 __asm__ __volatile__( 195 __asm__ __volatile__(
196 " testl %0, %0\n" 196 " testl %0, %0\n"
diff --git a/arch/x86/lib/usercopy_64.c b/arch/x86/lib/usercopy_64.c
index f4df6e7c718b..64d6c84e6353 100644
--- a/arch/x86/lib/usercopy_64.c
+++ b/arch/x86/lib/usercopy_64.c
@@ -15,7 +15,7 @@
15#define __do_strncpy_from_user(dst,src,count,res) \ 15#define __do_strncpy_from_user(dst,src,count,res) \
16do { \ 16do { \
17 long __d0, __d1, __d2; \ 17 long __d0, __d1, __d2; \
18 might_sleep(); \ 18 might_fault(); \
19 __asm__ __volatile__( \ 19 __asm__ __volatile__( \
20 " testq %1,%1\n" \ 20 " testq %1,%1\n" \
21 " jz 2f\n" \ 21 " jz 2f\n" \
@@ -64,7 +64,7 @@ EXPORT_SYMBOL(strncpy_from_user);
64unsigned long __clear_user(void __user *addr, unsigned long size) 64unsigned long __clear_user(void __user *addr, unsigned long size)
65{ 65{
66 long __d0; 66 long __d0;
67 might_sleep(); 67 might_fault();
68 /* no memory constraint because it doesn't change any memory gcc knows 68 /* no memory constraint because it doesn't change any memory gcc knows
69 about */ 69 about */
70 asm volatile( 70 asm volatile(
diff --git a/include/asm-um/system-generic.h b/include/asm-um/system-generic.h
index 5bcfa35e7a22..f1ea4da34fad 100644
--- a/include/asm-um/system-generic.h
+++ b/include/asm-um/system-generic.h
@@ -4,15 +4,15 @@
4#include "asm/arch/system.h" 4#include "asm/arch/system.h"
5 5
6#undef switch_to 6#undef switch_to
7#undef local_irq_save 7#undef raw_local_irq_save
8#undef local_irq_restore 8#undef raw_local_irq_restore
9#undef local_irq_disable 9#undef raw_local_irq_disable
10#undef local_irq_enable 10#undef raw_local_irq_enable
11#undef local_save_flags 11#undef raw_local_save_flags
12#undef local_irq_restore 12#undef raw_local_irq_restore
13#undef local_irq_enable 13#undef raw_local_irq_enable
14#undef local_irq_disable 14#undef raw_local_irq_disable
15#undef local_irq_save 15#undef raw_local_irq_save
16#undef irqs_disabled 16#undef irqs_disabled
17 17
18extern void *switch_to(void *prev, void *next, void *last); 18extern void *switch_to(void *prev, void *next, void *last);
@@ -23,21 +23,21 @@ extern int get_signals(void);
23extern void block_signals(void); 23extern void block_signals(void);
24extern void unblock_signals(void); 24extern void unblock_signals(void);
25 25
26#define local_save_flags(flags) do { typecheck(unsigned long, flags); \ 26#define raw_local_save_flags(flags) do { typecheck(unsigned long, flags); \
27 (flags) = get_signals(); } while(0) 27 (flags) = get_signals(); } while(0)
28#define local_irq_restore(flags) do { typecheck(unsigned long, flags); \ 28#define raw_local_irq_restore(flags) do { typecheck(unsigned long, flags); \
29 set_signals(flags); } while(0) 29 set_signals(flags); } while(0)
30 30
31#define local_irq_save(flags) do { local_save_flags(flags); \ 31#define raw_local_irq_save(flags) do { raw_local_save_flags(flags); \
32 local_irq_disable(); } while(0) 32 raw_local_irq_disable(); } while(0)
33 33
34#define local_irq_enable() unblock_signals() 34#define raw_local_irq_enable() unblock_signals()
35#define local_irq_disable() block_signals() 35#define raw_local_irq_disable() block_signals()
36 36
37#define irqs_disabled() \ 37#define irqs_disabled() \
38({ \ 38({ \
39 unsigned long flags; \ 39 unsigned long flags; \
40 local_save_flags(flags); \ 40 raw_local_save_flags(flags); \
41 (flags == 0); \ 41 (flags == 0); \
42}) 42})
43 43
diff --git a/include/asm-x86/uaccess.h b/include/asm-x86/uaccess.h
index 5f702d1d5218..dc8edb5c4659 100644
--- a/include/asm-x86/uaccess.h
+++ b/include/asm-x86/uaccess.h
@@ -157,6 +157,7 @@ extern int __get_user_bad(void);
157 int __ret_gu; \ 157 int __ret_gu; \
158 unsigned long __val_gu; \ 158 unsigned long __val_gu; \
159 __chk_user_ptr(ptr); \ 159 __chk_user_ptr(ptr); \
160 might_fault(); \
160 switch (sizeof(*(ptr))) { \ 161 switch (sizeof(*(ptr))) { \
161 case 1: \ 162 case 1: \
162 __get_user_x(1, __ret_gu, __val_gu, ptr); \ 163 __get_user_x(1, __ret_gu, __val_gu, ptr); \
@@ -241,6 +242,7 @@ extern void __put_user_8(void);
241 int __ret_pu; \ 242 int __ret_pu; \
242 __typeof__(*(ptr)) __pu_val; \ 243 __typeof__(*(ptr)) __pu_val; \
243 __chk_user_ptr(ptr); \ 244 __chk_user_ptr(ptr); \
245 might_fault(); \
244 __pu_val = x; \ 246 __pu_val = x; \
245 switch (sizeof(*(ptr))) { \ 247 switch (sizeof(*(ptr))) { \
246 case 1: \ 248 case 1: \
diff --git a/include/asm-x86/uaccess_32.h b/include/asm-x86/uaccess_32.h
index 6fdef39a0bcb..d10e842ec3ee 100644
--- a/include/asm-x86/uaccess_32.h
+++ b/include/asm-x86/uaccess_32.h
@@ -82,8 +82,8 @@ __copy_to_user_inatomic(void __user *to, const void *from, unsigned long n)
82static __always_inline unsigned long __must_check 82static __always_inline unsigned long __must_check
83__copy_to_user(void __user *to, const void *from, unsigned long n) 83__copy_to_user(void __user *to, const void *from, unsigned long n)
84{ 84{
85 might_sleep(); 85 might_fault();
86 return __copy_to_user_inatomic(to, from, n); 86 return __copy_to_user_inatomic(to, from, n);
87} 87}
88 88
89static __always_inline unsigned long 89static __always_inline unsigned long
@@ -137,7 +137,7 @@ __copy_from_user_inatomic(void *to, const void __user *from, unsigned long n)
137static __always_inline unsigned long 137static __always_inline unsigned long
138__copy_from_user(void *to, const void __user *from, unsigned long n) 138__copy_from_user(void *to, const void __user *from, unsigned long n)
139{ 139{
140 might_sleep(); 140 might_fault();
141 if (__builtin_constant_p(n)) { 141 if (__builtin_constant_p(n)) {
142 unsigned long ret; 142 unsigned long ret;
143 143
@@ -159,7 +159,7 @@ __copy_from_user(void *to, const void __user *from, unsigned long n)
159static __always_inline unsigned long __copy_from_user_nocache(void *to, 159static __always_inline unsigned long __copy_from_user_nocache(void *to,
160 const void __user *from, unsigned long n) 160 const void __user *from, unsigned long n)
161{ 161{
162 might_sleep(); 162 might_fault();
163 if (__builtin_constant_p(n)) { 163 if (__builtin_constant_p(n)) {
164 unsigned long ret; 164 unsigned long ret;
165 165
diff --git a/include/asm-x86/uaccess_64.h b/include/asm-x86/uaccess_64.h
index 515d4dce96b5..13fd56fbc3ab 100644
--- a/include/asm-x86/uaccess_64.h
+++ b/include/asm-x86/uaccess_64.h
@@ -28,6 +28,8 @@ static __always_inline __must_check
28int __copy_from_user(void *dst, const void __user *src, unsigned size) 28int __copy_from_user(void *dst, const void __user *src, unsigned size)
29{ 29{
30 int ret = 0; 30 int ret = 0;
31
32 might_fault();
31 if (!__builtin_constant_p(size)) 33 if (!__builtin_constant_p(size))
32 return copy_user_generic(dst, (__force void *)src, size); 34 return copy_user_generic(dst, (__force void *)src, size);
33 switch (size) { 35 switch (size) {
@@ -70,6 +72,8 @@ static __always_inline __must_check
70int __copy_to_user(void __user *dst, const void *src, unsigned size) 72int __copy_to_user(void __user *dst, const void *src, unsigned size)
71{ 73{
72 int ret = 0; 74 int ret = 0;
75
76 might_fault();
73 if (!__builtin_constant_p(size)) 77 if (!__builtin_constant_p(size))
74 return copy_user_generic((__force void *)dst, src, size); 78 return copy_user_generic((__force void *)dst, src, size);
75 switch (size) { 79 switch (size) {
@@ -112,6 +116,8 @@ static __always_inline __must_check
112int __copy_in_user(void __user *dst, const void __user *src, unsigned size) 116int __copy_in_user(void __user *dst, const void __user *src, unsigned size)
113{ 117{
114 int ret = 0; 118 int ret = 0;
119
120 might_fault();
115 if (!__builtin_constant_p(size)) 121 if (!__builtin_constant_p(size))
116 return copy_user_generic((__force void *)dst, 122 return copy_user_generic((__force void *)dst,
117 (__force void *)src, size); 123 (__force void *)src, size);
diff --git a/include/linux/debug_locks.h b/include/linux/debug_locks.h
index 4aaa4afb1cb9..096476f1fb35 100644
--- a/include/linux/debug_locks.h
+++ b/include/linux/debug_locks.h
@@ -17,7 +17,7 @@ extern int debug_locks_off(void);
17({ \ 17({ \
18 int __ret = 0; \ 18 int __ret = 0; \
19 \ 19 \
20 if (unlikely(c)) { \ 20 if (!oops_in_progress && unlikely(c)) { \
21 if (debug_locks_off() && !debug_locks_silent) \ 21 if (debug_locks_off() && !debug_locks_silent) \
22 WARN_ON(1); \ 22 WARN_ON(1); \
23 __ret = 1; \ 23 __ret = 1; \
diff --git a/include/linux/kernel.h b/include/linux/kernel.h
index 2651f805ba6d..e580ec095765 100644
--- a/include/linux/kernel.h
+++ b/include/linux/kernel.h
@@ -140,6 +140,15 @@ extern int _cond_resched(void);
140 (__x < 0) ? -__x : __x; \ 140 (__x < 0) ? -__x : __x; \
141 }) 141 })
142 142
143#ifdef CONFIG_PROVE_LOCKING
144void might_fault(void);
145#else
146static inline void might_fault(void)
147{
148 might_sleep();
149}
150#endif
151
143extern struct atomic_notifier_head panic_notifier_list; 152extern struct atomic_notifier_head panic_notifier_list;
144extern long (*panic_blink)(long time); 153extern long (*panic_blink)(long time);
145NORET_TYPE void panic(const char * fmt, ...) 154NORET_TYPE void panic(const char * fmt, ...)
diff --git a/include/linux/lockdep.h b/include/linux/lockdep.h
index 331e5f1c2d8e..0aa657aa8a1e 100644
--- a/include/linux/lockdep.h
+++ b/include/linux/lockdep.h
@@ -480,4 +480,22 @@ static inline void print_irqtrace_events(struct task_struct *curr)
480# define lock_map_release(l) do { } while (0) 480# define lock_map_release(l) do { } while (0)
481#endif 481#endif
482 482
483#ifdef CONFIG_PROVE_LOCKING
484# define might_lock(lock) \
485do { \
486 typecheck(struct lockdep_map *, &(lock)->dep_map); \
487 lock_acquire(&(lock)->dep_map, 0, 0, 0, 2, NULL, _THIS_IP_); \
488 lock_release(&(lock)->dep_map, 0, _THIS_IP_); \
489} while (0)
490# define might_lock_read(lock) \
491do { \
492 typecheck(struct lockdep_map *, &(lock)->dep_map); \
493 lock_acquire(&(lock)->dep_map, 0, 0, 1, 2, NULL, _THIS_IP_); \
494 lock_release(&(lock)->dep_map, 0, _THIS_IP_); \
495} while (0)
496#else
497# define might_lock(lock) do { } while (0)
498# define might_lock_read(lock) do { } while (0)
499#endif
500
483#endif /* __LINUX_LOCKDEP_H */ 501#endif /* __LINUX_LOCKDEP_H */
diff --git a/include/linux/uaccess.h b/include/linux/uaccess.h
index fec6decfb983..6b58367d145e 100644
--- a/include/linux/uaccess.h
+++ b/include/linux/uaccess.h
@@ -78,7 +78,7 @@ static inline unsigned long __copy_from_user_nocache(void *to,
78 \ 78 \
79 set_fs(KERNEL_DS); \ 79 set_fs(KERNEL_DS); \
80 pagefault_disable(); \ 80 pagefault_disable(); \
81 ret = __get_user(retval, (__force typeof(retval) __user *)(addr)); \ 81 ret = __copy_from_user_inatomic(&(retval), (__force typeof(retval) __user *)(addr), sizeof(retval)); \
82 pagefault_enable(); \ 82 pagefault_enable(); \
83 set_fs(old_fs); \ 83 set_fs(old_fs); \
84 ret; \ 84 ret; \
diff --git a/mm/memory.c b/mm/memory.c
index 1002f473f497..b8fdf4e5e65b 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -3016,3 +3016,18 @@ void print_vma_addr(char *prefix, unsigned long ip)
3016 } 3016 }
3017 up_read(&current->mm->mmap_sem); 3017 up_read(&current->mm->mmap_sem);
3018} 3018}
3019
3020#ifdef CONFIG_PROVE_LOCKING
3021void might_fault(void)
3022{
3023 might_sleep();
3024 /*
3025 * it would be nicer only to annotate paths which are not under
3026 * pagefault_disable, however that requires a larger audit and
3027 * providing helpers like get_user_atomic.
3028 */
3029 if (!in_atomic() && current->mm)
3030 might_lock_read(&current->mm->mmap_sem);
3031}
3032EXPORT_SYMBOL(might_fault);
3033#endif
generated by cgit v1.2.2 (git 2.25.0) at 2025-02-15 19:36:58 -0500