Merge branch 'for-4.19-nmi' into for-linus

3 files changed, 80 insertions, 44 deletions

diff --git a/kernel/printk/internal.h b/kernel/printk/internal.h
index 2a7d04049af4..0f1898820cba 100644
--- a/kernel/printk/internal.h
+++ b/kernel/printk/internal.h
@@ -19,11 +19,16 @@
 #ifdef CONFIG_PRINTK
 
 #define PRINTK_SAFE_CONTEXT_MASK         0x3fffffff
-#define PRINTK_NMI_DEFERRED_CONTEXT_MASK 0x40000000
+#define PRINTK_NMI_DIRECT_CONTEXT_MASK   0x40000000
 #define PRINTK_NMI_CONTEXT_MASK          0x80000000
 
 extern raw_spinlock_t logbuf_lock;
 
+__printf(5, 0)
+int vprintk_store(int facility, int level,
+                  const char *dict, size_t dictlen,
+                  const char *fmt, va_list args);
+
 __printf(1, 0) int vprintk_default(const char *fmt, va_list args);
 __printf(1, 0) int vprintk_deferred(const char *fmt, va_list args);
 __printf(1, 0) int vprintk_func(const char *fmt, va_list args);
@@ -54,6 +59,8 @@ void __printk_safe_exit(void);
                 local_irq_enable();             \
         } while (0)
 
+void defer_console_output(void);
+
 #else
 
 __printf(1, 0) int vprintk_func(const char *fmt, va_list args) { return 0; }
diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c
index e2cb0fc18e2d..9a63aeeaaf5d 100644
--- a/kernel/printk/printk.c
+++ b/kernel/printk/printk.c
@@ -1829,28 +1829,16 @@ static size_t log_output(int facility, int level, enum log_flags lflags, const c
         return log_store(facility, level, lflags, 0, dict, dictlen, text, text_len);
 }
 
-asmlinkage int vprintk_emit(int facility, int level,
-                            const char *dict, size_t dictlen,
-                            const char *fmt, va_list args)
+/* Must be called under logbuf_lock. */
+int vprintk_store(int facility, int level,
+                  const char *dict, size_t dictlen,
+                  const char *fmt, va_list args)
 {
         static char textbuf[LOG_LINE_MAX];
         char *text = textbuf;
         size_t text_len;
         enum log_flags lflags = 0;
-        unsigned long flags;
-        int printed_len;
-        bool in_sched = false;
-
-        if (level == LOGLEVEL_SCHED) {
-                level = LOGLEVEL_DEFAULT;
-                in_sched = true;
-        }
-
-        boot_delay_msec(level);
-        printk_delay();
 
-        /* This stops the holder of console_sem just where we want him */
-        logbuf_lock_irqsave(flags);
         /*
          * The printf needs to come first; we need the syslog
          * prefix which might be passed-in as a parameter.
@@ -1894,8 +1882,29 @@ asmlinkage int vprintk_emit(int facility, int level,
         if (suppress_message_printing(level))
                 lflags |= LOG_NOCONS;
 
-        printed_len = log_output(facility, level, lflags, dict, dictlen, text, text_len);
+        return log_output(facility, level, lflags,
+                          dict, dictlen, text, text_len);
+}
 
+asmlinkage int vprintk_emit(int facility, int level,
+                            const char *dict, size_t dictlen,
+                            const char *fmt, va_list args)
+{
+        int printed_len;
+        bool in_sched = false;
+        unsigned long flags;
+
+        if (level == LOGLEVEL_SCHED) {
+                level = LOGLEVEL_DEFAULT;
+                in_sched = true;
+        }
+
+        boot_delay_msec(level);
+        printk_delay();
+
+        /* This stops the holder of console_sem just where we want him */
+        logbuf_lock_irqsave(flags);
+        printed_len = vprintk_store(facility, level, dict, dictlen, fmt, args);
         logbuf_unlock_irqrestore(flags);
 
         /* If called from the scheduler, we can not call up(). */
@@ -2884,16 +2893,20 @@ void wake_up_klogd(void)
         preempt_enable();
 }
 
-int vprintk_deferred(const char *fmt, va_list args)
+void defer_console_output(void)
 {
-        int r;
-
-        r = vprintk_emit(0, LOGLEVEL_SCHED, NULL, 0, fmt, args);
-
         preempt_disable();
         __this_cpu_or(printk_pending, PRINTK_PENDING_OUTPUT);
         irq_work_queue(this_cpu_ptr(&wake_up_klogd_work));
         preempt_enable();
+}
+
+int vprintk_deferred(const char *fmt, va_list args)
+{
+        int r;
+
+        r = vprintk_emit(0, LOGLEVEL_SCHED, NULL, 0, fmt, args);
+        defer_console_output();
 
         return r;
 }
diff --git a/kernel/printk/printk_safe.c b/kernel/printk/printk_safe.c
index d7d091309054..a0a74c533e4b 100644
--- a/kernel/printk/printk_safe.c
+++ b/kernel/printk/printk_safe.c
@@ -308,24 +308,33 @@ static __printf(1, 0) int vprintk_nmi(const char *fmt, va_list args)
 
 void printk_nmi_enter(void)
 {
-        /*
-         * The size of the extra per-CPU buffer is limited. Use it only when
-         * the main one is locked. If this CPU is not in the safe context,
-         * the lock must be taken on another CPU and we could wait for it.
-         */
-        if ((this_cpu_read(printk_context) & PRINTK_SAFE_CONTEXT_MASK) &&
-            raw_spin_is_locked(&logbuf_lock)) {
-                this_cpu_or(printk_context, PRINTK_NMI_CONTEXT_MASK);
-        } else {
-                this_cpu_or(printk_context, PRINTK_NMI_DEFERRED_CONTEXT_MASK);
-        }
+        this_cpu_or(printk_context, PRINTK_NMI_CONTEXT_MASK);
 }
 
 void printk_nmi_exit(void)
 {
-        this_cpu_and(printk_context,
-                     ~(PRINTK_NMI_CONTEXT_MASK |
-                       PRINTK_NMI_DEFERRED_CONTEXT_MASK));
+        this_cpu_and(printk_context, ~PRINTK_NMI_CONTEXT_MASK);
+}
+
+/*
+ * Marks a code that might produce many messages in NMI context
+ * and the risk of losing them is more critical than eventual
+ * reordering.
+ *
+ * It has effect only when called in NMI context. Then printk()
+ * will try to store the messages into the main logbuf directly
+ * and use the per-CPU buffers only as a fallback when the lock
+ * is not available.
+ */
+void printk_nmi_direct_enter(void)
+{
+        if (this_cpu_read(printk_context) & PRINTK_NMI_CONTEXT_MASK)
+                this_cpu_or(printk_context, PRINTK_NMI_DIRECT_CONTEXT_MASK);
+}
+
+void printk_nmi_direct_exit(void)
+{
+        this_cpu_and(printk_context, ~PRINTK_NMI_DIRECT_CONTEXT_MASK);
 }
 
 #else
@@ -363,6 +372,20 @@ void __printk_safe_exit(void)
 
 __printf(1, 0) int vprintk_func(const char *fmt, va_list args)
 {
+        /*
+         * Try to use the main logbuf even in NMI. But avoid calling console
+         * drivers that might have their own locks.
+         */
+        if ((this_cpu_read(printk_context) & PRINTK_NMI_DIRECT_CONTEXT_MASK) &&
+            raw_spin_trylock(&logbuf_lock)) {
+                int len;
+
+                len = vprintk_store(0, LOGLEVEL_DEFAULT, NULL, 0, fmt, args);
+                raw_spin_unlock(&logbuf_lock);
+                defer_console_output();
+                return len;
+        }
+
         /* Use extra buffer in NMI when logbuf_lock is taken or in safe mode. */
         if (this_cpu_read(printk_context) & PRINTK_NMI_CONTEXT_MASK)
                 return vprintk_nmi(fmt, args);
@@ -371,13 +394,6 @@ __printf(1, 0) int vprintk_func(const char *fmt, va_list args)
         if (this_cpu_read(printk_context) & PRINTK_SAFE_CONTEXT_MASK)
                 return vprintk_safe(fmt, args);
 
-        /*
-         * Use the main logbuf when logbuf_lock is available in NMI.
-         * But avoid calling console drivers that might have their own locks.
-         */
-        if (this_cpu_read(printk_context) & PRINTK_NMI_DEFERRED_CONTEXT_MASK)
-                return vprintk_deferred(fmt, args);
-
         /* No obstacles. */
         return vprintk_default(fmt, args);
 }