aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--MAINTAINERS8
-rw-r--r--include/linux/printk.h21
-rw-r--r--init/Kconfig16
-rw-r--r--init/main.c2
-rw-r--r--kernel/kexec_core.c2
-rw-r--r--kernel/panic.c4
-rw-r--r--kernel/printk/Makefile2
-rw-r--r--kernel/printk/internal.h79
-rw-r--r--kernel/printk/printk.c232
-rw-r--r--kernel/printk/printk_safe.c (renamed from kernel/printk/nmi.c)234
-rw-r--r--lib/nmi_backtrace.c2
11 files changed, 347 insertions, 255 deletions
diff --git a/MAINTAINERS b/MAINTAINERS
index 7757bef14951..ca6f5f7a4752 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -9997,6 +9997,14 @@ S: Supported
9997F: Documentation/preempt-locking.txt 9997F: Documentation/preempt-locking.txt
9998F: include/linux/preempt.h 9998F: include/linux/preempt.h
9999 9999
10000PRINTK
10001M: Petr Mladek <pmladek@suse.com>
10002M: Sergey Senozhatsky <sergey.senozhatsky@gmail.com>
10003R: Steven Rostedt <rostedt@goodmis.org>
10004S: Maintained
10005F: kernel/printk/
10006F: include/linux/printk.h
10007
10000PRISM54 WIRELESS DRIVER 10008PRISM54 WIRELESS DRIVER
10001M: "Luis R. Rodriguez" <mcgrof@gmail.com> 10009M: "Luis R. Rodriguez" <mcgrof@gmail.com>
10002L: linux-wireless@vger.kernel.org 10010L: linux-wireless@vger.kernel.org
diff --git a/include/linux/printk.h b/include/linux/printk.h
index 3472cc6b7a60..571257e0f53d 100644
--- a/include/linux/printk.h
+++ b/include/linux/printk.h
@@ -147,17 +147,11 @@ void early_printk(const char *s, ...) { }
147#endif 147#endif
148 148
149#ifdef CONFIG_PRINTK_NMI 149#ifdef CONFIG_PRINTK_NMI
150extern void printk_nmi_init(void);
151extern void printk_nmi_enter(void); 150extern void printk_nmi_enter(void);
152extern void printk_nmi_exit(void); 151extern void printk_nmi_exit(void);
153extern void printk_nmi_flush(void);
154extern void printk_nmi_flush_on_panic(void);
155#else 152#else
156static inline void printk_nmi_init(void) { }
157static inline void printk_nmi_enter(void) { } 153static inline void printk_nmi_enter(void) { }
158static inline void printk_nmi_exit(void) { } 154static inline void printk_nmi_exit(void) { }
159static inline void printk_nmi_flush(void) { }
160static inline void printk_nmi_flush_on_panic(void) { }
161#endif /* PRINTK_NMI */ 155#endif /* PRINTK_NMI */
162 156
163#ifdef CONFIG_PRINTK 157#ifdef CONFIG_PRINTK
@@ -209,6 +203,9 @@ void __init setup_log_buf(int early);
209__printf(1, 2) void dump_stack_set_arch_desc(const char *fmt, ...); 203__printf(1, 2) void dump_stack_set_arch_desc(const char *fmt, ...);
210void dump_stack_print_info(const char *log_lvl); 204void dump_stack_print_info(const char *log_lvl);
211void show_regs_print_info(const char *log_lvl); 205void show_regs_print_info(const char *log_lvl);
206extern void printk_safe_init(void);
207extern void printk_safe_flush(void);
208extern void printk_safe_flush_on_panic(void);
212#else 209#else
213static inline __printf(1, 0) 210static inline __printf(1, 0)
214int vprintk(const char *s, va_list args) 211int vprintk(const char *s, va_list args)
@@ -268,6 +265,18 @@ static inline void dump_stack_print_info(const char *log_lvl)
268static inline void show_regs_print_info(const char *log_lvl) 265static inline void show_regs_print_info(const char *log_lvl)
269{ 266{
270} 267}
268
269static inline void printk_safe_init(void)
270{
271}
272
273static inline void printk_safe_flush(void)
274{
275}
276
277static inline void printk_safe_flush_on_panic(void)
278{
279}
271#endif 280#endif
272 281
273extern asmlinkage void dump_stack(void) __cold; 282extern asmlinkage void dump_stack(void) __cold;
diff --git a/init/Kconfig b/init/Kconfig
index 55bb6fbc294e..483ad679aa37 100644
--- a/init/Kconfig
+++ b/init/Kconfig
@@ -861,17 +861,19 @@ config LOG_CPU_MAX_BUF_SHIFT
861 13 => 8 KB for each CPU 861 13 => 8 KB for each CPU
862 12 => 4 KB for each CPU 862 12 => 4 KB for each CPU
863 863
864config NMI_LOG_BUF_SHIFT 864config PRINTK_SAFE_LOG_BUF_SHIFT
865 int "Temporary per-CPU NMI log buffer size (12 => 4KB, 13 => 8KB)" 865 int "Temporary per-CPU printk log buffer size (12 => 4KB, 13 => 8KB)"
866 range 10 21 866 range 10 21
867 default 13 867 default 13
868 depends on PRINTK_NMI 868 depends on PRINTK
869 help 869 help
870 Select the size of a per-CPU buffer where NMI messages are temporary 870 Select the size of an alternate printk per-CPU buffer where messages
871 stored. They are copied to the main log buffer in a safe context 871 printed from usafe contexts are temporary stored. One example would
872 to avoid a deadlock. The value defines the size as a power of 2. 872 be NMI messages, another one - printk recursion. The messages are
873 copied to the main log buffer in a safe context to avoid a deadlock.
874 The value defines the size as a power of 2.
873 875
874 NMI messages are rare and limited. The largest one is when 876 Those messages are rare and limited. The largest one is when
875 a backtrace is printed. It usually fits into 4KB. Select 877 a backtrace is printed. It usually fits into 4KB. Select
876 8KB if you want to be on the safe side. 878 8KB if you want to be on the safe side.
877 879
diff --git a/init/main.c b/init/main.c
index c8a00f0f10ff..24ea48745061 100644
--- a/init/main.c
+++ b/init/main.c
@@ -581,7 +581,7 @@ asmlinkage __visible void __init start_kernel(void)
581 timekeeping_init(); 581 timekeeping_init();
582 time_init(); 582 time_init();
583 sched_clock_postinit(); 583 sched_clock_postinit();
584 printk_nmi_init(); 584 printk_safe_init();
585 perf_event_init(); 585 perf_event_init();
586 profile_init(); 586 profile_init();
587 call_function_init(); 587 call_function_init();
diff --git a/kernel/kexec_core.c b/kernel/kexec_core.c
index a01974e1bf6b..bfe62d5b3872 100644
--- a/kernel/kexec_core.c
+++ b/kernel/kexec_core.c
@@ -916,7 +916,7 @@ void crash_kexec(struct pt_regs *regs)
916 old_cpu = atomic_cmpxchg(&panic_cpu, PANIC_CPU_INVALID, this_cpu); 916 old_cpu = atomic_cmpxchg(&panic_cpu, PANIC_CPU_INVALID, this_cpu);
917 if (old_cpu == PANIC_CPU_INVALID) { 917 if (old_cpu == PANIC_CPU_INVALID) {
918 /* This is the 1st CPU which comes here, so go ahead. */ 918 /* This is the 1st CPU which comes here, so go ahead. */
919 printk_nmi_flush_on_panic(); 919 printk_safe_flush_on_panic();
920 __crash_kexec(regs); 920 __crash_kexec(regs);
921 921
922 /* 922 /*
diff --git a/kernel/panic.c b/kernel/panic.c
index 08aa88dde7de..b95959733ce0 100644
--- a/kernel/panic.c
+++ b/kernel/panic.c
@@ -188,7 +188,7 @@ void panic(const char *fmt, ...)
188 * Bypass the panic_cpu check and call __crash_kexec directly. 188 * Bypass the panic_cpu check and call __crash_kexec directly.
189 */ 189 */
190 if (!_crash_kexec_post_notifiers) { 190 if (!_crash_kexec_post_notifiers) {
191 printk_nmi_flush_on_panic(); 191 printk_safe_flush_on_panic();
192 __crash_kexec(NULL); 192 __crash_kexec(NULL);
193 193
194 /* 194 /*
@@ -213,7 +213,7 @@ void panic(const char *fmt, ...)
213 atomic_notifier_call_chain(&panic_notifier_list, 0, buf); 213 atomic_notifier_call_chain(&panic_notifier_list, 0, buf);
214 214
215 /* Call flush even twice. It tries harder with a single online CPU */ 215 /* Call flush even twice. It tries harder with a single online CPU */
216 printk_nmi_flush_on_panic(); 216 printk_safe_flush_on_panic();
217 kmsg_dump(KMSG_DUMP_PANIC); 217 kmsg_dump(KMSG_DUMP_PANIC);
218 218
219 /* 219 /*
diff --git a/kernel/printk/Makefile b/kernel/printk/Makefile
index abb0042a427b..4a2ffc39eb95 100644
--- a/kernel/printk/Makefile
+++ b/kernel/printk/Makefile
@@ -1,3 +1,3 @@
1obj-y = printk.o 1obj-y = printk.o
2obj-$(CONFIG_PRINTK_NMI) += nmi.o 2obj-$(CONFIG_PRINTK) += printk_safe.o
3obj-$(CONFIG_A11Y_BRAILLE_CONSOLE) += braille.o 3obj-$(CONFIG_A11Y_BRAILLE_CONSOLE) += braille.o
diff --git a/kernel/printk/internal.h b/kernel/printk/internal.h
index 7fd2838fa417..1db044f808b7 100644
--- a/kernel/printk/internal.h
+++ b/kernel/printk/internal.h
@@ -16,42 +16,55 @@
16 */ 16 */
17#include <linux/percpu.h> 17#include <linux/percpu.h>
18 18
19typedef __printf(1, 0) int (*printk_func_t)(const char *fmt, va_list args); 19#ifdef CONFIG_PRINTK
20 20
21int __printf(1, 0) vprintk_default(const char *fmt, va_list args); 21#define PRINTK_SAFE_CONTEXT_MASK 0x7fffffff
22 22#define PRINTK_NMI_CONTEXT_MASK 0x80000000
23#ifdef CONFIG_PRINTK_NMI
24 23
25extern raw_spinlock_t logbuf_lock; 24extern raw_spinlock_t logbuf_lock;
26 25
26__printf(1, 0) int vprintk_default(const char *fmt, va_list args);
27__printf(1, 0) int vprintk_func(const char *fmt, va_list args);
28void __printk_safe_enter(void);
29void __printk_safe_exit(void);
30
31#define printk_safe_enter_irqsave(flags) \
32 do { \
33 local_irq_save(flags); \
34 __printk_safe_enter(); \
35 } while (0)
36
37#define printk_safe_exit_irqrestore(flags) \
38 do { \
39 __printk_safe_exit(); \
40 local_irq_restore(flags); \
41 } while (0)
42
43#define printk_safe_enter_irq() \
44 do { \
45 local_irq_disable(); \
46 __printk_safe_enter(); \
47 } while (0)
48
49#define printk_safe_exit_irq() \
50 do { \
51 __printk_safe_exit(); \
52 local_irq_enable(); \
53 } while (0)
54
55#else
56
57__printf(1, 0) int vprintk_func(const char *fmt, va_list args) { return 0; }
58
27/* 59/*
28 * printk() could not take logbuf_lock in NMI context. Instead, 60 * In !PRINTK builds we still export logbuf_lock spin_lock, console_sem
29 * it temporary stores the strings into a per-CPU buffer. 61 * semaphore and some of console functions (console_unlock()/etc.), so
30 * The alternative implementation is chosen transparently 62 * printk-safe must preserve the existing local IRQ guarantees.
31 * via per-CPU variable.
32 */ 63 */
33DECLARE_PER_CPU(printk_func_t, printk_func); 64#define printk_safe_enter_irqsave(flags) local_irq_save(flags)
34static inline __printf(1, 0) int vprintk_func(const char *fmt, va_list args) 65#define printk_safe_exit_irqrestore(flags) local_irq_restore(flags)
35{ 66
36 return this_cpu_read(printk_func)(fmt, args); 67#define printk_safe_enter_irq() local_irq_disable()
37} 68#define printk_safe_exit_irq() local_irq_enable()
38 69
39extern atomic_t nmi_message_lost; 70#endif /* CONFIG_PRINTK */
40static inline int get_nmi_message_lost(void)
41{
42 return atomic_xchg(&nmi_message_lost, 0);
43}
44
45#else /* CONFIG_PRINTK_NMI */
46
47static inline __printf(1, 0) int vprintk_func(const char *fmt, va_list args)
48{
49 return vprintk_default(fmt, args);
50}
51
52static inline int get_nmi_message_lost(void)
53{
54 return 0;
55}
56
57#endif /* CONFIG_PRINTK_NMI */
diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c
index 4ba3d34938c0..34da86e73d00 100644
--- a/kernel/printk/printk.c
+++ b/kernel/printk/printk.c
@@ -213,17 +213,36 @@ static int nr_ext_console_drivers;
213 213
214static int __down_trylock_console_sem(unsigned long ip) 214static int __down_trylock_console_sem(unsigned long ip)
215{ 215{
216 if (down_trylock(&console_sem)) 216 int lock_failed;
217 unsigned long flags;
218
219 /*
220 * Here and in __up_console_sem() we need to be in safe mode,
221 * because spindump/WARN/etc from under console ->lock will
222 * deadlock in printk()->down_trylock_console_sem() otherwise.
223 */
224 printk_safe_enter_irqsave(flags);
225 lock_failed = down_trylock(&console_sem);
226 printk_safe_exit_irqrestore(flags);
227
228 if (lock_failed)
217 return 1; 229 return 1;
218 mutex_acquire(&console_lock_dep_map, 0, 1, ip); 230 mutex_acquire(&console_lock_dep_map, 0, 1, ip);
219 return 0; 231 return 0;
220} 232}
221#define down_trylock_console_sem() __down_trylock_console_sem(_RET_IP_) 233#define down_trylock_console_sem() __down_trylock_console_sem(_RET_IP_)
222 234
223#define up_console_sem() do { \ 235static void __up_console_sem(unsigned long ip)
224 mutex_release(&console_lock_dep_map, 1, _RET_IP_);\ 236{
225 up(&console_sem);\ 237 unsigned long flags;
226} while (0) 238
239 mutex_release(&console_lock_dep_map, 1, ip);
240
241 printk_safe_enter_irqsave(flags);
242 up(&console_sem);
243 printk_safe_exit_irqrestore(flags);
244}
245#define up_console_sem() __up_console_sem(_RET_IP_)
227 246
228/* 247/*
229 * This is used for debugging the mess that is the VT code by 248 * This is used for debugging the mess that is the VT code by
@@ -351,6 +370,34 @@ __packed __aligned(4)
351 */ 370 */
352DEFINE_RAW_SPINLOCK(logbuf_lock); 371DEFINE_RAW_SPINLOCK(logbuf_lock);
353 372
373/*
374 * Helper macros to lock/unlock logbuf_lock and switch between
375 * printk-safe/unsafe modes.
376 */
377#define logbuf_lock_irq() \
378 do { \
379 printk_safe_enter_irq(); \
380 raw_spin_lock(&logbuf_lock); \
381 } while (0)
382
383#define logbuf_unlock_irq() \
384 do { \
385 raw_spin_unlock(&logbuf_lock); \
386 printk_safe_exit_irq(); \
387 } while (0)
388
389#define logbuf_lock_irqsave(flags) \
390 do { \
391 printk_safe_enter_irqsave(flags); \
392 raw_spin_lock(&logbuf_lock); \
393 } while (0)
394
395#define logbuf_unlock_irqrestore(flags) \
396 do { \
397 raw_spin_unlock(&logbuf_lock); \
398 printk_safe_exit_irqrestore(flags); \
399 } while (0)
400
354#ifdef CONFIG_PRINTK 401#ifdef CONFIG_PRINTK
355DECLARE_WAIT_QUEUE_HEAD(log_wait); 402DECLARE_WAIT_QUEUE_HEAD(log_wait);
356/* the next printk record to read by syslog(READ) or /proc/kmsg */ 403/* the next printk record to read by syslog(READ) or /proc/kmsg */
@@ -782,20 +829,21 @@ static ssize_t devkmsg_read(struct file *file, char __user *buf,
782 ret = mutex_lock_interruptible(&user->lock); 829 ret = mutex_lock_interruptible(&user->lock);
783 if (ret) 830 if (ret)
784 return ret; 831 return ret;
785 raw_spin_lock_irq(&logbuf_lock); 832
833 logbuf_lock_irq();
786 while (user->seq == log_next_seq) { 834 while (user->seq == log_next_seq) {
787 if (file->f_flags & O_NONBLOCK) { 835 if (file->f_flags & O_NONBLOCK) {
788 ret = -EAGAIN; 836 ret = -EAGAIN;
789 raw_spin_unlock_irq(&logbuf_lock); 837 logbuf_unlock_irq();
790 goto out; 838 goto out;
791 } 839 }
792 840
793 raw_spin_unlock_irq(&logbuf_lock); 841 logbuf_unlock_irq();
794 ret = wait_event_interruptible(log_wait, 842 ret = wait_event_interruptible(log_wait,
795 user->seq != log_next_seq); 843 user->seq != log_next_seq);
796 if (ret) 844 if (ret)
797 goto out; 845 goto out;
798 raw_spin_lock_irq(&logbuf_lock); 846 logbuf_lock_irq();
799 } 847 }
800 848
801 if (user->seq < log_first_seq) { 849 if (user->seq < log_first_seq) {
@@ -803,7 +851,7 @@ static ssize_t devkmsg_read(struct file *file, char __user *buf,
803 user->idx = log_first_idx; 851 user->idx = log_first_idx;
804 user->seq = log_first_seq; 852 user->seq = log_first_seq;
805 ret = -EPIPE; 853 ret = -EPIPE;
806 raw_spin_unlock_irq(&logbuf_lock); 854 logbuf_unlock_irq();
807 goto out; 855 goto out;
808 } 856 }
809 857
@@ -816,7 +864,7 @@ static ssize_t devkmsg_read(struct file *file, char __user *buf,
816 864
817 user->idx = log_next(user->idx); 865 user->idx = log_next(user->idx);
818 user->seq++; 866 user->seq++;
819 raw_spin_unlock_irq(&logbuf_lock); 867 logbuf_unlock_irq();
820 868
821 if (len > count) { 869 if (len > count) {
822 ret = -EINVAL; 870 ret = -EINVAL;
@@ -843,7 +891,7 @@ static loff_t devkmsg_llseek(struct file *file, loff_t offset, int whence)
843 if (offset) 891 if (offset)
844 return -ESPIPE; 892 return -ESPIPE;
845 893
846 raw_spin_lock_irq(&logbuf_lock); 894 logbuf_lock_irq();
847 switch (whence) { 895 switch (whence) {
848 case SEEK_SET: 896 case SEEK_SET:
849 /* the first record */ 897 /* the first record */
@@ -867,7 +915,7 @@ static loff_t devkmsg_llseek(struct file *file, loff_t offset, int whence)
867 default: 915 default:
868 ret = -EINVAL; 916 ret = -EINVAL;
869 } 917 }
870 raw_spin_unlock_irq(&logbuf_lock); 918 logbuf_unlock_irq();
871 return ret; 919 return ret;
872} 920}
873 921
@@ -881,7 +929,7 @@ static unsigned int devkmsg_poll(struct file *file, poll_table *wait)
881 929
882 poll_wait(file, &log_wait, wait); 930 poll_wait(file, &log_wait, wait);
883 931
884 raw_spin_lock_irq(&logbuf_lock); 932 logbuf_lock_irq();
885 if (user->seq < log_next_seq) { 933 if (user->seq < log_next_seq) {
886 /* return error when data has vanished underneath us */ 934 /* return error when data has vanished underneath us */
887 if (user->seq < log_first_seq) 935 if (user->seq < log_first_seq)
@@ -889,7 +937,7 @@ static unsigned int devkmsg_poll(struct file *file, poll_table *wait)
889 else 937 else
890 ret = POLLIN|POLLRDNORM; 938 ret = POLLIN|POLLRDNORM;
891 } 939 }
892 raw_spin_unlock_irq(&logbuf_lock); 940 logbuf_unlock_irq();
893 941
894 return ret; 942 return ret;
895} 943}
@@ -919,10 +967,10 @@ static int devkmsg_open(struct inode *inode, struct file *file)
919 967
920 mutex_init(&user->lock); 968 mutex_init(&user->lock);
921 969
922 raw_spin_lock_irq(&logbuf_lock); 970 logbuf_lock_irq();
923 user->idx = log_first_idx; 971 user->idx = log_first_idx;
924 user->seq = log_first_seq; 972 user->seq = log_first_seq;
925 raw_spin_unlock_irq(&logbuf_lock); 973 logbuf_unlock_irq();
926 974
927 file->private_data = user; 975 file->private_data = user;
928 return 0; 976 return 0;
@@ -1064,13 +1112,13 @@ void __init setup_log_buf(int early)
1064 return; 1112 return;
1065 } 1113 }
1066 1114
1067 raw_spin_lock_irqsave(&logbuf_lock, flags); 1115 logbuf_lock_irqsave(flags);
1068 log_buf_len = new_log_buf_len; 1116 log_buf_len = new_log_buf_len;
1069 log_buf = new_log_buf; 1117 log_buf = new_log_buf;
1070 new_log_buf_len = 0; 1118 new_log_buf_len = 0;
1071 free = __LOG_BUF_LEN - log_next_idx; 1119 free = __LOG_BUF_LEN - log_next_idx;
1072 memcpy(log_buf, __log_buf, __LOG_BUF_LEN); 1120 memcpy(log_buf, __log_buf, __LOG_BUF_LEN);
1073 raw_spin_unlock_irqrestore(&logbuf_lock, flags); 1121 logbuf_unlock_irqrestore(flags);
1074 1122
1075 pr_info("log_buf_len: %d bytes\n", log_buf_len); 1123 pr_info("log_buf_len: %d bytes\n", log_buf_len);
1076 pr_info("early log buf free: %d(%d%%)\n", 1124 pr_info("early log buf free: %d(%d%%)\n",
@@ -1248,7 +1296,7 @@ static int syslog_print(char __user *buf, int size)
1248 size_t n; 1296 size_t n;
1249 size_t skip; 1297 size_t skip;
1250 1298
1251 raw_spin_lock_irq(&logbuf_lock); 1299 logbuf_lock_irq();
1252 if (syslog_seq < log_first_seq) { 1300 if (syslog_seq < log_first_seq) {
1253 /* messages are gone, move to first one */ 1301 /* messages are gone, move to first one */
1254 syslog_seq = log_first_seq; 1302 syslog_seq = log_first_seq;
@@ -1256,7 +1304,7 @@ static int syslog_print(char __user *buf, int size)
1256 syslog_partial = 0; 1304 syslog_partial = 0;
1257 } 1305 }
1258 if (syslog_seq == log_next_seq) { 1306 if (syslog_seq == log_next_seq) {
1259 raw_spin_unlock_irq(&logbuf_lock); 1307 logbuf_unlock_irq();
1260 break; 1308 break;
1261 } 1309 }
1262 1310
@@ -1275,7 +1323,7 @@ static int syslog_print(char __user *buf, int size)
1275 syslog_partial += n; 1323 syslog_partial += n;
1276 } else 1324 } else
1277 n = 0; 1325 n = 0;
1278 raw_spin_unlock_irq(&logbuf_lock); 1326 logbuf_unlock_irq();
1279 1327
1280 if (!n) 1328 if (!n)
1281 break; 1329 break;
@@ -1304,7 +1352,7 @@ static int syslog_print_all(char __user *buf, int size, bool clear)
1304 if (!text) 1352 if (!text)
1305 return -ENOMEM; 1353 return -ENOMEM;
1306 1354
1307 raw_spin_lock_irq(&logbuf_lock); 1355 logbuf_lock_irq();
1308 if (buf) { 1356 if (buf) {
1309 u64 next_seq; 1357 u64 next_seq;
1310 u64 seq; 1358 u64 seq;
@@ -1352,12 +1400,12 @@ static int syslog_print_all(char __user *buf, int size, bool clear)
1352 idx = log_next(idx); 1400 idx = log_next(idx);
1353 seq++; 1401 seq++;
1354 1402
1355 raw_spin_unlock_irq(&logbuf_lock); 1403 logbuf_unlock_irq();
1356 if (copy_to_user(buf + len, text, textlen)) 1404 if (copy_to_user(buf + len, text, textlen))
1357 len = -EFAULT; 1405 len = -EFAULT;
1358 else 1406 else
1359 len += textlen; 1407 len += textlen;
1360 raw_spin_lock_irq(&logbuf_lock); 1408 logbuf_lock_irq();
1361 1409
1362 if (seq < log_first_seq) { 1410 if (seq < log_first_seq) {
1363 /* messages are gone, move to next one */ 1411 /* messages are gone, move to next one */
@@ -1371,7 +1419,7 @@ static int syslog_print_all(char __user *buf, int size, bool clear)
1371 clear_seq = log_next_seq; 1419 clear_seq = log_next_seq;
1372 clear_idx = log_next_idx; 1420 clear_idx = log_next_idx;
1373 } 1421 }
1374 raw_spin_unlock_irq(&logbuf_lock); 1422 logbuf_unlock_irq();
1375 1423
1376 kfree(text); 1424 kfree(text);
1377 return len; 1425 return len;
@@ -1458,7 +1506,7 @@ int do_syslog(int type, char __user *buf, int len, int source)
1458 break; 1506 break;
1459 /* Number of chars in the log buffer */ 1507 /* Number of chars in the log buffer */
1460 case SYSLOG_ACTION_SIZE_UNREAD: 1508 case SYSLOG_ACTION_SIZE_UNREAD:
1461 raw_spin_lock_irq(&logbuf_lock); 1509 logbuf_lock_irq();
1462 if (syslog_seq < log_first_seq) { 1510 if (syslog_seq < log_first_seq) {
1463 /* messages are gone, move to first one */ 1511 /* messages are gone, move to first one */
1464 syslog_seq = log_first_seq; 1512 syslog_seq = log_first_seq;
@@ -1486,7 +1534,7 @@ int do_syslog(int type, char __user *buf, int len, int source)
1486 } 1534 }
1487 error -= syslog_partial; 1535 error -= syslog_partial;
1488 } 1536 }
1489 raw_spin_unlock_irq(&logbuf_lock); 1537 logbuf_unlock_irq();
1490 break; 1538 break;
1491 /* Size of the log buffer */ 1539 /* Size of the log buffer */
1492 case SYSLOG_ACTION_SIZE_BUFFER: 1540 case SYSLOG_ACTION_SIZE_BUFFER:
@@ -1510,8 +1558,7 @@ SYSCALL_DEFINE3(syslog, int, type, char __user *, buf, int, len)
1510 * log_buf[start] to log_buf[end - 1]. 1558 * log_buf[start] to log_buf[end - 1].
1511 * The console_lock must be held. 1559 * The console_lock must be held.
1512 */ 1560 */
1513static void call_console_drivers(int level, 1561static void call_console_drivers(const char *ext_text, size_t ext_len,
1514 const char *ext_text, size_t ext_len,
1515 const char *text, size_t len) 1562 const char *text, size_t len)
1516{ 1563{
1517 struct console *con; 1564 struct console *con;
@@ -1538,28 +1585,6 @@ static void call_console_drivers(int level,
1538 } 1585 }
1539} 1586}
1540 1587
1541/*
1542 * Zap console related locks when oopsing.
1543 * To leave time for slow consoles to print a full oops,
1544 * only zap at most once every 30 seconds.
1545 */
1546static void zap_locks(void)
1547{
1548 static unsigned long oops_timestamp;
1549
1550 if (time_after_eq(jiffies, oops_timestamp) &&
1551 !time_after(jiffies, oops_timestamp + 30 * HZ))
1552 return;
1553
1554 oops_timestamp = jiffies;
1555
1556 debug_locks_off();
1557 /* If a crash is occurring, make sure we can't deadlock */
1558 raw_spin_lock_init(&logbuf_lock);
1559 /* And make sure that we print immediately */
1560 sema_init(&console_sem, 1);
1561}
1562
1563int printk_delay_msec __read_mostly; 1588int printk_delay_msec __read_mostly;
1564 1589
1565static inline void printk_delay(void) 1590static inline void printk_delay(void)
@@ -1669,18 +1694,13 @@ asmlinkage int vprintk_emit(int facility, int level,
1669 const char *dict, size_t dictlen, 1694 const char *dict, size_t dictlen,
1670 const char *fmt, va_list args) 1695 const char *fmt, va_list args)
1671{ 1696{
1672 static bool recursion_bug;
1673 static char textbuf[LOG_LINE_MAX]; 1697 static char textbuf[LOG_LINE_MAX];
1674 char *text = textbuf; 1698 char *text = textbuf;
1675 size_t text_len = 0; 1699 size_t text_len = 0;
1676 enum log_flags lflags = 0; 1700 enum log_flags lflags = 0;
1677 unsigned long flags; 1701 unsigned long flags;
1678 int this_cpu;
1679 int printed_len = 0; 1702 int printed_len = 0;
1680 int nmi_message_lost;
1681 bool in_sched = false; 1703 bool in_sched = false;
1682 /* cpu currently holding logbuf_lock in this function */
1683 static unsigned int logbuf_cpu = UINT_MAX;
1684 1704
1685 if (level == LOGLEVEL_SCHED) { 1705 if (level == LOGLEVEL_SCHED) {
1686 level = LOGLEVEL_DEFAULT; 1706 level = LOGLEVEL_DEFAULT;
@@ -1690,53 +1710,8 @@ asmlinkage int vprintk_emit(int facility, int level,
1690 boot_delay_msec(level); 1710 boot_delay_msec(level);
1691 printk_delay(); 1711 printk_delay();
1692 1712
1693 local_irq_save(flags);
1694 this_cpu = smp_processor_id();
1695
1696 /*
1697 * Ouch, printk recursed into itself!
1698 */
1699 if (unlikely(logbuf_cpu == this_cpu)) {
1700 /*
1701 * If a crash is occurring during printk() on this CPU,
1702 * then try to get the crash message out but make sure
1703 * we can't deadlock. Otherwise just return to avoid the
1704 * recursion and return - but flag the recursion so that
1705 * it can be printed at the next appropriate moment:
1706 */
1707 if (!oops_in_progress && !lockdep_recursing(current)) {
1708 recursion_bug = true;
1709 local_irq_restore(flags);
1710 return 0;
1711 }
1712 zap_locks();
1713 }
1714
1715 lockdep_off();
1716 /* This stops the holder of console_sem just where we want him */ 1713 /* This stops the holder of console_sem just where we want him */
1717 raw_spin_lock(&logbuf_lock); 1714 logbuf_lock_irqsave(flags);
1718 logbuf_cpu = this_cpu;
1719
1720 if (unlikely(recursion_bug)) {
1721 static const char recursion_msg[] =
1722 "BUG: recent printk recursion!";
1723
1724 recursion_bug = false;
1725 /* emit KERN_CRIT message */
1726 printed_len += log_store(0, 2, LOG_PREFIX|LOG_NEWLINE, 0,
1727 NULL, 0, recursion_msg,
1728 strlen(recursion_msg));
1729 }
1730
1731 nmi_message_lost = get_nmi_message_lost();
1732 if (unlikely(nmi_message_lost)) {
1733 text_len = scnprintf(textbuf, sizeof(textbuf),
1734 "BAD LUCK: lost %d message(s) from NMI context!",
1735 nmi_message_lost);
1736 printed_len += log_store(0, 2, LOG_PREFIX|LOG_NEWLINE, 0,
1737 NULL, 0, textbuf, text_len);
1738 }
1739
1740 /* 1715 /*
1741 * The printf needs to come first; we need the syslog 1716 * The printf needs to come first; we need the syslog
1742 * prefix which might be passed-in as a parameter. 1717 * prefix which might be passed-in as a parameter.
@@ -1779,14 +1754,10 @@ asmlinkage int vprintk_emit(int facility, int level,
1779 1754
1780 printed_len += log_output(facility, level, lflags, dict, dictlen, text, text_len); 1755 printed_len += log_output(facility, level, lflags, dict, dictlen, text, text_len);
1781 1756
1782 logbuf_cpu = UINT_MAX; 1757 logbuf_unlock_irqrestore(flags);
1783 raw_spin_unlock(&logbuf_lock);
1784 lockdep_on();
1785 local_irq_restore(flags);
1786 1758
1787 /* If called from the scheduler, we can not call up(). */ 1759 /* If called from the scheduler, we can not call up(). */
1788 if (!in_sched) { 1760 if (!in_sched) {
1789 lockdep_off();
1790 /* 1761 /*
1791 * Try to acquire and then immediately release the console 1762 * Try to acquire and then immediately release the console
1792 * semaphore. The release will print out buffers and wake up 1763 * semaphore. The release will print out buffers and wake up
@@ -1794,7 +1765,6 @@ asmlinkage int vprintk_emit(int facility, int level,
1794 */ 1765 */
1795 if (console_trylock()) 1766 if (console_trylock())
1796 console_unlock(); 1767 console_unlock();
1797 lockdep_on();
1798 } 1768 }
1799 1769
1800 return printed_len; 1770 return printed_len;
@@ -1803,7 +1773,7 @@ EXPORT_SYMBOL(vprintk_emit);
1803 1773
1804asmlinkage int vprintk(const char *fmt, va_list args) 1774asmlinkage int vprintk(const char *fmt, va_list args)
1805{ 1775{
1806 return vprintk_emit(0, LOGLEVEL_DEFAULT, NULL, 0, fmt, args); 1776 return vprintk_func(fmt, args);
1807} 1777}
1808EXPORT_SYMBOL(vprintk); 1778EXPORT_SYMBOL(vprintk);
1809 1779
@@ -1895,16 +1865,12 @@ static ssize_t msg_print_ext_header(char *buf, size_t size,
1895static ssize_t msg_print_ext_body(char *buf, size_t size, 1865static ssize_t msg_print_ext_body(char *buf, size_t size,
1896 char *dict, size_t dict_len, 1866 char *dict, size_t dict_len,
1897 char *text, size_t text_len) { return 0; } 1867 char *text, size_t text_len) { return 0; }
1898static void call_console_drivers(int level, 1868static void call_console_drivers(const char *ext_text, size_t ext_len,
1899 const char *ext_text, size_t ext_len,
1900 const char *text, size_t len) {} 1869 const char *text, size_t len) {}
1901static size_t msg_print_text(const struct printk_log *msg, 1870static size_t msg_print_text(const struct printk_log *msg,
1902 bool syslog, char *buf, size_t size) { return 0; } 1871 bool syslog, char *buf, size_t size) { return 0; }
1903static bool suppress_message_printing(int level) { return false; } 1872static bool suppress_message_printing(int level) { return false; }
1904 1873
1905/* Still needs to be defined for users */
1906DEFINE_PER_CPU(printk_func_t, printk_func);
1907
1908#endif /* CONFIG_PRINTK */ 1874#endif /* CONFIG_PRINTK */
1909 1875
1910#ifdef CONFIG_EARLY_PRINTK 1876#ifdef CONFIG_EARLY_PRINTK
@@ -2220,9 +2186,9 @@ again:
2220 struct printk_log *msg; 2186 struct printk_log *msg;
2221 size_t ext_len = 0; 2187 size_t ext_len = 0;
2222 size_t len; 2188 size_t len;
2223 int level;
2224 2189
2225 raw_spin_lock_irqsave(&logbuf_lock, flags); 2190 printk_safe_enter_irqsave(flags);
2191 raw_spin_lock(&logbuf_lock);
2226 if (seen_seq != log_next_seq) { 2192 if (seen_seq != log_next_seq) {
2227 wake_klogd = true; 2193 wake_klogd = true;
2228 seen_seq = log_next_seq; 2194 seen_seq = log_next_seq;
@@ -2243,8 +2209,7 @@ skip:
2243 break; 2209 break;
2244 2210
2245 msg = log_from_idx(console_idx); 2211 msg = log_from_idx(console_idx);
2246 level = msg->level; 2212 if (suppress_message_printing(msg->level)) {
2247 if (suppress_message_printing(level)) {
2248 /* 2213 /*
2249 * Skip record we have buffered and already printed 2214 * Skip record we have buffered and already printed
2250 * directly to the console when we received it, and 2215 * directly to the console when we received it, and
@@ -2270,9 +2235,9 @@ skip:
2270 raw_spin_unlock(&logbuf_lock); 2235 raw_spin_unlock(&logbuf_lock);
2271 2236
2272 stop_critical_timings(); /* don't trace print latency */ 2237 stop_critical_timings(); /* don't trace print latency */
2273 call_console_drivers(level, ext_text, ext_len, text, len); 2238 call_console_drivers(ext_text, ext_len, text, len);
2274 start_critical_timings(); 2239 start_critical_timings();
2275 local_irq_restore(flags); 2240 printk_safe_exit_irqrestore(flags);
2276 2241
2277 if (do_cond_resched) 2242 if (do_cond_resched)
2278 cond_resched(); 2243 cond_resched();
@@ -2295,7 +2260,8 @@ skip:
2295 */ 2260 */
2296 raw_spin_lock(&logbuf_lock); 2261 raw_spin_lock(&logbuf_lock);
2297 retry = console_seq != log_next_seq; 2262 retry = console_seq != log_next_seq;
2298 raw_spin_unlock_irqrestore(&logbuf_lock, flags); 2263 raw_spin_unlock(&logbuf_lock);
2264 printk_safe_exit_irqrestore(flags);
2299 2265
2300 if (retry && console_trylock()) 2266 if (retry && console_trylock())
2301 goto again; 2267 goto again;
@@ -2558,10 +2524,10 @@ void register_console(struct console *newcon)
2558 * console_unlock(); will print out the buffered messages 2524 * console_unlock(); will print out the buffered messages
2559 * for us. 2525 * for us.
2560 */ 2526 */
2561 raw_spin_lock_irqsave(&logbuf_lock, flags); 2527 logbuf_lock_irqsave(flags);
2562 console_seq = syslog_seq; 2528 console_seq = syslog_seq;
2563 console_idx = syslog_idx; 2529 console_idx = syslog_idx;
2564 raw_spin_unlock_irqrestore(&logbuf_lock, flags); 2530 logbuf_unlock_irqrestore(flags);
2565 /* 2531 /*
2566 * We're about to replay the log buffer. Only do this to the 2532 * We're about to replay the log buffer. Only do this to the
2567 * just-registered console to avoid excessive message spam to 2533 * just-registered console to avoid excessive message spam to
@@ -2860,12 +2826,12 @@ void kmsg_dump(enum kmsg_dump_reason reason)
2860 /* initialize iterator with data about the stored records */ 2826 /* initialize iterator with data about the stored records */
2861 dumper->active = true; 2827 dumper->active = true;
2862 2828
2863 raw_spin_lock_irqsave(&logbuf_lock, flags); 2829 logbuf_lock_irqsave(flags);
2864 dumper->cur_seq = clear_seq; 2830 dumper->cur_seq = clear_seq;
2865 dumper->cur_idx = clear_idx; 2831 dumper->cur_idx = clear_idx;
2866 dumper->next_seq = log_next_seq; 2832 dumper->next_seq = log_next_seq;
2867 dumper->next_idx = log_next_idx; 2833 dumper->next_idx = log_next_idx;
2868 raw_spin_unlock_irqrestore(&logbuf_lock, flags); 2834 logbuf_unlock_irqrestore(flags);
2869 2835
2870 /* invoke dumper which will iterate over records */ 2836 /* invoke dumper which will iterate over records */
2871 dumper->dump(dumper, reason); 2837 dumper->dump(dumper, reason);
@@ -2950,9 +2916,9 @@ bool kmsg_dump_get_line(struct kmsg_dumper *dumper, bool syslog,
2950 unsigned long flags; 2916 unsigned long flags;
2951 bool ret; 2917 bool ret;
2952 2918
2953 raw_spin_lock_irqsave(&logbuf_lock, flags); 2919 logbuf_lock_irqsave(flags);
2954 ret = kmsg_dump_get_line_nolock(dumper, syslog, line, size, len); 2920 ret = kmsg_dump_get_line_nolock(dumper, syslog, line, size, len);
2955 raw_spin_unlock_irqrestore(&logbuf_lock, flags); 2921 logbuf_unlock_irqrestore(flags);
2956 2922
2957 return ret; 2923 return ret;
2958} 2924}
@@ -2991,7 +2957,7 @@ bool kmsg_dump_get_buffer(struct kmsg_dumper *dumper, bool syslog,
2991 if (!dumper->active) 2957 if (!dumper->active)
2992 goto out; 2958 goto out;
2993 2959
2994 raw_spin_lock_irqsave(&logbuf_lock, flags); 2960 logbuf_lock_irqsave(flags);
2995 if (dumper->cur_seq < log_first_seq) { 2961 if (dumper->cur_seq < log_first_seq) {
2996 /* messages are gone, move to first available one */ 2962 /* messages are gone, move to first available one */
2997 dumper->cur_seq = log_first_seq; 2963 dumper->cur_seq = log_first_seq;
@@ -3000,7 +2966,7 @@ bool kmsg_dump_get_buffer(struct kmsg_dumper *dumper, bool syslog,
3000 2966
3001 /* last entry */ 2967 /* last entry */
3002 if (dumper->cur_seq >= dumper->next_seq) { 2968 if (dumper->cur_seq >= dumper->next_seq) {
3003 raw_spin_unlock_irqrestore(&logbuf_lock, flags); 2969 logbuf_unlock_irqrestore(flags);
3004 goto out; 2970 goto out;
3005 } 2971 }
3006 2972
@@ -3042,7 +3008,7 @@ bool kmsg_dump_get_buffer(struct kmsg_dumper *dumper, bool syslog,
3042 dumper->next_seq = next_seq; 3008 dumper->next_seq = next_seq;
3043 dumper->next_idx = next_idx; 3009 dumper->next_idx = next_idx;
3044 ret = true; 3010 ret = true;
3045 raw_spin_unlock_irqrestore(&logbuf_lock, flags); 3011 logbuf_unlock_irqrestore(flags);
3046out: 3012out:
3047 if (len) 3013 if (len)
3048 *len = l; 3014 *len = l;
@@ -3080,9 +3046,9 @@ void kmsg_dump_rewind(struct kmsg_dumper *dumper)
3080{ 3046{
3081 unsigned long flags; 3047 unsigned long flags;
3082 3048
3083 raw_spin_lock_irqsave(&logbuf_lock, flags); 3049 logbuf_lock_irqsave(flags);
3084 kmsg_dump_rewind_nolock(dumper); 3050 kmsg_dump_rewind_nolock(dumper);
3085 raw_spin_unlock_irqrestore(&logbuf_lock, flags); 3051 logbuf_unlock_irqrestore(flags);
3086} 3052}
3087EXPORT_SYMBOL_GPL(kmsg_dump_rewind); 3053EXPORT_SYMBOL_GPL(kmsg_dump_rewind);
3088 3054
diff --git a/kernel/printk/nmi.c b/kernel/printk/printk_safe.c
index f011aaef583c..033e50a7d706 100644
--- a/kernel/printk/nmi.c
+++ b/kernel/printk/printk_safe.c
@@ -1,5 +1,5 @@
1/* 1/*
2 * nmi.c - Safe printk in NMI context 2 * printk_safe.c - Safe printk for printk-deadlock-prone contexts
3 * 3 *
4 * This program is free software; you can redistribute it and/or 4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License 5 * modify it under the terms of the GNU General Public License
@@ -32,36 +32,58 @@
32 * is later flushed into the main ring buffer via IRQ work. 32 * is later flushed into the main ring buffer via IRQ work.
33 * 33 *
34 * The alternative implementation is chosen transparently 34 * The alternative implementation is chosen transparently
35 * via @printk_func per-CPU variable. 35 * by examinig current printk() context mask stored in @printk_context
36 * per-CPU variable.
36 * 37 *
37 * The implementation allows to flush the strings also from another CPU. 38 * The implementation allows to flush the strings also from another CPU.
38 * There are situations when we want to make sure that all buffers 39 * There are situations when we want to make sure that all buffers
39 * were handled or when IRQs are blocked. 40 * were handled or when IRQs are blocked.
40 */ 41 */
41DEFINE_PER_CPU(printk_func_t, printk_func) = vprintk_default; 42static int printk_safe_irq_ready;
42static int printk_nmi_irq_ready;
43atomic_t nmi_message_lost;
44 43
45#define NMI_LOG_BUF_LEN ((1 << CONFIG_NMI_LOG_BUF_SHIFT) - \ 44#define SAFE_LOG_BUF_LEN ((1 << CONFIG_PRINTK_SAFE_LOG_BUF_SHIFT) - \
46 sizeof(atomic_t) - sizeof(struct irq_work)) 45 sizeof(atomic_t) - \
46 sizeof(atomic_t) - \
47 sizeof(struct irq_work))
47 48
48struct nmi_seq_buf { 49struct printk_safe_seq_buf {
49 atomic_t len; /* length of written data */ 50 atomic_t len; /* length of written data */
51 atomic_t message_lost;
50 struct irq_work work; /* IRQ work that flushes the buffer */ 52 struct irq_work work; /* IRQ work that flushes the buffer */
51 unsigned char buffer[NMI_LOG_BUF_LEN]; 53 unsigned char buffer[SAFE_LOG_BUF_LEN];
52}; 54};
53static DEFINE_PER_CPU(struct nmi_seq_buf, nmi_print_seq); 55
56static DEFINE_PER_CPU(struct printk_safe_seq_buf, safe_print_seq);
57static DEFINE_PER_CPU(int, printk_context);
58
59#ifdef CONFIG_PRINTK_NMI
60static DEFINE_PER_CPU(struct printk_safe_seq_buf, nmi_print_seq);
61#endif
62
63/* Get flushed in a more safe context. */
64static void queue_flush_work(struct printk_safe_seq_buf *s)
65{
66 if (printk_safe_irq_ready) {
67 /* Make sure that IRQ work is really initialized. */
68 smp_rmb();
69 irq_work_queue(&s->work);
70 }
71}
54 72
55/* 73/*
56 * Safe printk() for NMI context. It uses a per-CPU buffer to 74 * Add a message to per-CPU context-dependent buffer. NMI and printk-safe
57 * store the message. NMIs are not nested, so there is always only 75 * have dedicated buffers, because otherwise printk-safe preempted by
58 * one writer running. But the buffer might get flushed from another 76 * NMI-printk would have overwritten the NMI messages.
59 * CPU, so we need to be careful. 77 *
78 * The messages are fushed from irq work (or from panic()), possibly,
79 * from other CPU, concurrently with printk_safe_log_store(). Should this
80 * happen, printk_safe_log_store() will notice the buffer->len mismatch
81 * and repeat the write.
60 */ 82 */
61static int vprintk_nmi(const char *fmt, va_list args) 83static int printk_safe_log_store(struct printk_safe_seq_buf *s,
84 const char *fmt, va_list args)
62{ 85{
63 struct nmi_seq_buf *s = this_cpu_ptr(&nmi_print_seq); 86 int add;
64 int add = 0;
65 size_t len; 87 size_t len;
66 88
67again: 89again:
@@ -69,18 +91,21 @@ again:
69 91
70 /* The trailing '\0' is not counted into len. */ 92 /* The trailing '\0' is not counted into len. */
71 if (len >= sizeof(s->buffer) - 1) { 93 if (len >= sizeof(s->buffer) - 1) {
72 atomic_inc(&nmi_message_lost); 94 atomic_inc(&s->message_lost);
95 queue_flush_work(s);
73 return 0; 96 return 0;
74 } 97 }
75 98
76 /* 99 /*
77 * Make sure that all old data have been read before the buffer was 100 * Make sure that all old data have been read before the buffer
78 * reseted. This is not needed when we just append data. 101 * was reset. This is not needed when we just append data.
79 */ 102 */
80 if (!len) 103 if (!len)
81 smp_rmb(); 104 smp_rmb();
82 105
83 add = vscnprintf(s->buffer + len, sizeof(s->buffer) - len, fmt, args); 106 add = vscnprintf(s->buffer + len, sizeof(s->buffer) - len, fmt, args);
107 if (!add)
108 return 0;
84 109
85 /* 110 /*
86 * Do it once again if the buffer has been flushed in the meantime. 111 * Do it once again if the buffer has been flushed in the meantime.
@@ -90,32 +115,23 @@ again:
90 if (atomic_cmpxchg(&s->len, len, len + add) != len) 115 if (atomic_cmpxchg(&s->len, len, len + add) != len)
91 goto again; 116 goto again;
92 117
93 /* Get flushed in a more safe context. */ 118 queue_flush_work(s);
94 if (add && printk_nmi_irq_ready) {
95 /* Make sure that IRQ work is really initialized. */
96 smp_rmb();
97 irq_work_queue(&s->work);
98 }
99
100 return add; 119 return add;
101} 120}
102 121
103static void printk_nmi_flush_line(const char *text, int len) 122static inline void printk_safe_flush_line(const char *text, int len)
104{ 123{
105 /* 124 /*
106 * The buffers are flushed in NMI only on panic. The messages must 125 * Avoid any console drivers calls from here, because we may be
107 * go only into the ring buffer at this stage. Consoles will get 126 * in NMI or printk_safe context (when in panic). The messages
108 * explicitly called later when a crashdump is not generated. 127 * must go only into the ring buffer at this stage. Consoles will
128 * get explicitly called later when a crashdump is not generated.
109 */ 129 */
110 if (in_nmi()) 130 printk_deferred("%.*s", len, text);
111 printk_deferred("%.*s", len, text);
112 else
113 printk("%.*s", len, text);
114
115} 131}
116 132
117/* printk part of the temporary buffer line by line */ 133/* printk part of the temporary buffer line by line */
118static int printk_nmi_flush_buffer(const char *start, size_t len) 134static int printk_safe_flush_buffer(const char *start, size_t len)
119{ 135{
120 const char *c, *end; 136 const char *c, *end;
121 bool header; 137 bool header;
@@ -127,7 +143,7 @@ static int printk_nmi_flush_buffer(const char *start, size_t len)
127 /* Print line by line. */ 143 /* Print line by line. */
128 while (c < end) { 144 while (c < end) {
129 if (*c == '\n') { 145 if (*c == '\n') {
130 printk_nmi_flush_line(start, c - start + 1); 146 printk_safe_flush_line(start, c - start + 1);
131 start = ++c; 147 start = ++c;
132 header = true; 148 header = true;
133 continue; 149 continue;
@@ -140,7 +156,7 @@ static int printk_nmi_flush_buffer(const char *start, size_t len)
140 continue; 156 continue;
141 } 157 }
142 158
143 printk_nmi_flush_line(start, c - start); 159 printk_safe_flush_line(start, c - start);
144 start = c++; 160 start = c++;
145 header = true; 161 header = true;
146 continue; 162 continue;
@@ -154,22 +170,31 @@ static int printk_nmi_flush_buffer(const char *start, size_t len)
154 if (start < end && !header) { 170 if (start < end && !header) {
155 static const char newline[] = KERN_CONT "\n"; 171 static const char newline[] = KERN_CONT "\n";
156 172
157 printk_nmi_flush_line(start, end - start); 173 printk_safe_flush_line(start, end - start);
158 printk_nmi_flush_line(newline, strlen(newline)); 174 printk_safe_flush_line(newline, strlen(newline));
159 } 175 }
160 176
161 return len; 177 return len;
162} 178}
163 179
180static void report_message_lost(struct printk_safe_seq_buf *s)
181{
182 int lost = atomic_xchg(&s->message_lost, 0);
183
184 if (lost)
185 printk_deferred("Lost %d message(s)!\n", lost);
186}
187
164/* 188/*
165 * Flush data from the associated per_CPU buffer. The function 189 * Flush data from the associated per-CPU buffer. The function
166 * can be called either via IRQ work or independently. 190 * can be called either via IRQ work or independently.
167 */ 191 */
168static void __printk_nmi_flush(struct irq_work *work) 192static void __printk_safe_flush(struct irq_work *work)
169{ 193{
170 static raw_spinlock_t read_lock = 194 static raw_spinlock_t read_lock =
171 __RAW_SPIN_LOCK_INITIALIZER(read_lock); 195 __RAW_SPIN_LOCK_INITIALIZER(read_lock);
172 struct nmi_seq_buf *s = container_of(work, struct nmi_seq_buf, work); 196 struct printk_safe_seq_buf *s =
197 container_of(work, struct printk_safe_seq_buf, work);
173 unsigned long flags; 198 unsigned long flags;
174 size_t len; 199 size_t len;
175 int i; 200 int i;
@@ -194,9 +219,9 @@ more:
194 * buffer size. 219 * buffer size.
195 */ 220 */
196 if ((i && i >= len) || len > sizeof(s->buffer)) { 221 if ((i && i >= len) || len > sizeof(s->buffer)) {
197 const char *msg = "printk_nmi_flush: internal error\n"; 222 const char *msg = "printk_safe_flush: internal error\n";
198 223
199 printk_nmi_flush_line(msg, strlen(msg)); 224 printk_safe_flush_line(msg, strlen(msg));
200 len = 0; 225 len = 0;
201 } 226 }
202 227
@@ -205,7 +230,7 @@ more:
205 230
206 /* Make sure that data has been written up to the @len */ 231 /* Make sure that data has been written up to the @len */
207 smp_rmb(); 232 smp_rmb();
208 i += printk_nmi_flush_buffer(s->buffer + i, len - i); 233 i += printk_safe_flush_buffer(s->buffer + i, len - i);
209 234
210 /* 235 /*
211 * Check that nothing has got added in the meantime and truncate 236 * Check that nothing has got added in the meantime and truncate
@@ -217,35 +242,40 @@ more:
217 goto more; 242 goto more;
218 243
219out: 244out:
245 report_message_lost(s);
220 raw_spin_unlock_irqrestore(&read_lock, flags); 246 raw_spin_unlock_irqrestore(&read_lock, flags);
221} 247}
222 248
223/** 249/**
224 * printk_nmi_flush - flush all per-cpu nmi buffers. 250 * printk_safe_flush - flush all per-cpu nmi buffers.
225 * 251 *
226 * The buffers are flushed automatically via IRQ work. This function 252 * The buffers are flushed automatically via IRQ work. This function
227 * is useful only when someone wants to be sure that all buffers have 253 * is useful only when someone wants to be sure that all buffers have
228 * been flushed at some point. 254 * been flushed at some point.
229 */ 255 */
230void printk_nmi_flush(void) 256void printk_safe_flush(void)
231{ 257{
232 int cpu; 258 int cpu;
233 259
234 for_each_possible_cpu(cpu) 260 for_each_possible_cpu(cpu) {
235 __printk_nmi_flush(&per_cpu(nmi_print_seq, cpu).work); 261#ifdef CONFIG_PRINTK_NMI
262 __printk_safe_flush(&per_cpu(nmi_print_seq, cpu).work);
263#endif
264 __printk_safe_flush(&per_cpu(safe_print_seq, cpu).work);
265 }
236} 266}
237 267
238/** 268/**
239 * printk_nmi_flush_on_panic - flush all per-cpu nmi buffers when the system 269 * printk_safe_flush_on_panic - flush all per-cpu nmi buffers when the system
240 * goes down. 270 * goes down.
241 * 271 *
242 * Similar to printk_nmi_flush() but it can be called even in NMI context when 272 * Similar to printk_safe_flush() but it can be called even in NMI context when
243 * the system goes down. It does the best effort to get NMI messages into 273 * the system goes down. It does the best effort to get NMI messages into
244 * the main ring buffer. 274 * the main ring buffer.
245 * 275 *
246 * Note that it could try harder when there is only one CPU online. 276 * Note that it could try harder when there is only one CPU online.
247 */ 277 */
248void printk_nmi_flush_on_panic(void) 278void printk_safe_flush_on_panic(void)
249{ 279{
250 /* 280 /*
251 * Make sure that we could access the main ring buffer. 281 * Make sure that we could access the main ring buffer.
@@ -259,33 +289,97 @@ void printk_nmi_flush_on_panic(void)
259 raw_spin_lock_init(&logbuf_lock); 289 raw_spin_lock_init(&logbuf_lock);
260 } 290 }
261 291
262 printk_nmi_flush(); 292 printk_safe_flush();
263} 293}
264 294
265void __init printk_nmi_init(void) 295#ifdef CONFIG_PRINTK_NMI
296/*
297 * Safe printk() for NMI context. It uses a per-CPU buffer to
298 * store the message. NMIs are not nested, so there is always only
299 * one writer running. But the buffer might get flushed from another
300 * CPU, so we need to be careful.
301 */
302static int vprintk_nmi(const char *fmt, va_list args)
266{ 303{
267 int cpu; 304 struct printk_safe_seq_buf *s = this_cpu_ptr(&nmi_print_seq);
268 305
269 for_each_possible_cpu(cpu) { 306 return printk_safe_log_store(s, fmt, args);
270 struct nmi_seq_buf *s = &per_cpu(nmi_print_seq, cpu); 307}
271 308
272 init_irq_work(&s->work, __printk_nmi_flush); 309void printk_nmi_enter(void)
273 } 310{
311 this_cpu_or(printk_context, PRINTK_NMI_CONTEXT_MASK);
312}
274 313
275 /* Make sure that IRQ works are initialized before enabling. */ 314void printk_nmi_exit(void)
276 smp_wmb(); 315{
277 printk_nmi_irq_ready = 1; 316 this_cpu_and(printk_context, ~PRINTK_NMI_CONTEXT_MASK);
317}
278 318
279 /* Flush pending messages that did not have scheduled IRQ works. */ 319#else
280 printk_nmi_flush(); 320
321static int vprintk_nmi(const char *fmt, va_list args)
322{
323 return 0;
281} 324}
282 325
283void printk_nmi_enter(void) 326#endif /* CONFIG_PRINTK_NMI */
327
328/*
329 * Lock-less printk(), to avoid deadlocks should the printk() recurse
330 * into itself. It uses a per-CPU buffer to store the message, just like
331 * NMI.
332 */
333static int vprintk_safe(const char *fmt, va_list args)
284{ 334{
285 this_cpu_write(printk_func, vprintk_nmi); 335 struct printk_safe_seq_buf *s = this_cpu_ptr(&safe_print_seq);
336
337 return printk_safe_log_store(s, fmt, args);
286} 338}
287 339
288void printk_nmi_exit(void) 340/* Can be preempted by NMI. */
341void __printk_safe_enter(void)
342{
343 this_cpu_inc(printk_context);
344}
345
346/* Can be preempted by NMI. */
347void __printk_safe_exit(void)
289{ 348{
290 this_cpu_write(printk_func, vprintk_default); 349 this_cpu_dec(printk_context);
350}
351
352__printf(1, 0) int vprintk_func(const char *fmt, va_list args)
353{
354 if (this_cpu_read(printk_context) & PRINTK_NMI_CONTEXT_MASK)
355 return vprintk_nmi(fmt, args);
356
357 if (this_cpu_read(printk_context) & PRINTK_SAFE_CONTEXT_MASK)
358 return vprintk_safe(fmt, args);
359
360 return vprintk_default(fmt, args);
361}
362
363void __init printk_safe_init(void)
364{
365 int cpu;
366
367 for_each_possible_cpu(cpu) {
368 struct printk_safe_seq_buf *s;
369
370 s = &per_cpu(safe_print_seq, cpu);
371 init_irq_work(&s->work, __printk_safe_flush);
372
373#ifdef CONFIG_PRINTK_NMI
374 s = &per_cpu(nmi_print_seq, cpu);
375 init_irq_work(&s->work, __printk_safe_flush);
376#endif
377 }
378
379 /* Make sure that IRQ works are initialized before enabling. */
380 smp_wmb();
381 printk_safe_irq_ready = 1;
382
383 /* Flush pending messages that did not have scheduled IRQ works. */
384 printk_safe_flush();
291} 385}
diff --git a/lib/nmi_backtrace.c b/lib/nmi_backtrace.c
index 75554754eadf..5f7999eacad5 100644
--- a/lib/nmi_backtrace.c
+++ b/lib/nmi_backtrace.c
@@ -77,7 +77,7 @@ void nmi_trigger_cpumask_backtrace(const cpumask_t *mask,
77 * Force flush any remote buffers that might be stuck in IRQ context 77 * Force flush any remote buffers that might be stuck in IRQ context
78 * and therefore could not run their irq_work. 78 * and therefore could not run their irq_work.
79 */ 79 */
80 printk_nmi_flush(); 80 printk_safe_flush();
81 81
82 clear_bit_unlock(0, &backtrace_flag); 82 clear_bit_unlock(0, &backtrace_flag);
83 put_cpu(); 83 put_cpu();
generated by cgit v1.2.2 (git 2.25.0) at 2025-10-01 12:28:08 -0400