Merge branch 'x86-asm-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull low-level x86 updates from Ingo Molnar:
 "In this cycle this topic tree has become one of those 'super topics'
  that accumulated a lot of changes:

   - Add CONFIG_VMAP_STACK=y support to the core kernel and enable it on
     x86 - preceded by an array of changes. v4.8 saw preparatory changes
     in this area already - this is the rest of the work. Includes the
     thread stack caching performance optimization. (Andy Lutomirski)

   - switch_to() cleanups and all around enhancements. (Brian Gerst)

   - A large number of dumpstack infrastructure enhancements and an
     unwinder abstraction. The secret long term plan is safe(r) live
     patching plus maybe another attempt at debuginfo based unwinding -
     but all these current bits are standalone enhancements in a frame
     pointer based debug environment as well. (Josh Poimboeuf)

   - More __ro_after_init and const annotations. (Kees Cook)

   - Enable KASLR for the vmemmap memory region. (Thomas Garnier)"

[ The virtually mapped stack changes are pretty fundamental, and not
  x86-specific per se, even if they are only used on x86 right now. ]

* 'x86-asm-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (70 commits)
  x86/asm: Get rid of __read_cr4_safe()
  thread_info: Use unsigned long for flags
  x86/alternatives: Add stack frame dependency to alternative_call_2()
  x86/dumpstack: Fix show_stack() task pointer regression
  x86/dumpstack: Remove dump_trace() and related callbacks
  x86/dumpstack: Convert show_trace_log_lvl() to use the new unwinder
  oprofile/x86: Convert x86_backtrace() to use the new unwinder
  x86/stacktrace: Convert save_stack_trace_*() to use the new unwinder
  perf/x86: Convert perf_callchain_kernel() to use the new unwinder
  x86/unwind: Add new unwind interface and implementations
  x86/dumpstack: Remove NULL task pointer convention
  fork: Optimize task creation by caching two thread stacks per CPU if CONFIG_VMAP_STACK=y
  sched/core: Free the stack early if CONFIG_THREAD_INFO_IN_TASK
  lib/syscall: Pin the task stack in collect_syscall()
  x86/process: Pin the target stack in get_wchan()
  x86/dumpstack: Pin the target stack when dumping it
  kthread: Pin the stack via try_get_task_stack()/put_task_stack() in to_live_kthread() function
  sched/core: Add try_get_task_stack() and put_task_stack()
  x86/entry/64: Fix a minor comment rebase error
  iommu/amd: Don't put completion-wait semaphore on stack
  ...

1 files changed, 110 insertions, 148 deletions

diff --git a/arch/x86/kernel/dumpstack.c b/arch/x86/kernel/dumpstack.c
index 92e8f0a7159c..9b7cf5c28f5f 100644
--- a/arch/x86/kernel/dumpstack.c
+++ b/arch/x86/kernel/dumpstack.c
@@ -17,7 +17,7 @@
 #include <linux/sysfs.h>
 
 #include <asm/stacktrace.h>
-
+#include <asm/unwind.h>
 
 int panic_on_unrecovered_nmi;
 int panic_on_io_nmi;
@@ -25,11 +25,29 @@ unsigned int code_bytes = 64;
 int kstack_depth_to_print = 3 * STACKSLOTS_PER_LINE;
 static int die_counter;
 
+bool in_task_stack(unsigned long *stack, struct task_struct *task,
+                   struct stack_info *info)
+{
+        unsigned long *begin = task_stack_page(task);
+        unsigned long *end   = task_stack_page(task) + THREAD_SIZE;
+
+        if (stack < begin || stack >= end)
+                return false;
+
+        info->type      = STACK_TYPE_TASK;
+        info->begin     = begin;
+        info->end       = end;
+        info->next_sp   = NULL;
+
+        return true;
+}
+
 static void printk_stack_address(unsigned long address, int reliable,
-                void *data)
+                                 char *log_lvl)
 {
+        touch_nmi_watchdog();
         printk("%s [<%p>] %s%pB\n",
-                (char *)data, (void *)address, reliable ? "" : "? ",
+                log_lvl, (void *)address, reliable ? "" : "? ",
                 (void *)address);
 }
 
@@ -38,176 +56,120 @@ void printk_address(unsigned long address)
         pr_cont(" [<%p>] %pS\n", (void *)address, (void *)address);
 }
 
-#ifdef CONFIG_FUNCTION_GRAPH_TRACER
-static void
-print_ftrace_graph_addr(unsigned long addr, void *data,
-                        const struct stacktrace_ops *ops,
-                        struct task_struct *task, int *graph)
+void show_trace_log_lvl(struct task_struct *task, struct pt_regs *regs,
+                        unsigned long *stack, char *log_lvl)
 {
-        unsigned long ret_addr;
-        int index;
-
-        if (addr != (unsigned long)return_to_handler)
-                return;
-
-        index = task->curr_ret_stack;
-
-        if (!task->ret_stack || index < *graph)
-                return;
-
-        index -= *graph;
-        ret_addr = task->ret_stack[index].ret;
-
-        ops->address(data, ret_addr, 1);
+        struct unwind_state state;
+        struct stack_info stack_info = {0};
+        unsigned long visit_mask = 0;
+        int graph_idx = 0;
 
-        (*graph)++;
-}
-#else
-static inline void
-print_ftrace_graph_addr(unsigned long addr, void *data,
-                        const struct stacktrace_ops *ops,
-                        struct task_struct *task, int *graph)
-{ }
-#endif
-
-/*
- * x86-64 can have up to three kernel stacks:
- * process stack
- * interrupt stack
- * severe exception (double fault, nmi, stack fault, debug, mce) hardware stack
- */
-
-static inline int valid_stack_ptr(struct task_struct *task,
-                        void *p, unsigned int size, void *end)
-{
-        void *t = task_stack_page(task);
-        if (end) {
-                if (p < end && p >= (end-THREAD_SIZE))
-                        return 1;
-                else
-                        return 0;
-        }
-        return p >= t && p < t + THREAD_SIZE - size;
-}
+        printk("%sCall Trace:\n", log_lvl);
 
-unsigned long
-print_context_stack(struct task_struct *task,
-                unsigned long *stack, unsigned long bp,
-                const struct stacktrace_ops *ops, void *data,
-                unsigned long *end, int *graph)
-{
-        struct stack_frame *frame = (struct stack_frame *)bp;
+        unwind_start(&state, task, regs, stack);
 
         /*
-         * If we overflowed the stack into a guard page, jump back to the
-         * bottom of the usable stack.
+         * Iterate through the stacks, starting with the current stack pointer.
+         * Each stack has a pointer to the next one.
+         *
+         * x86-64 can have several stacks:
+         * - task stack
+         * - interrupt stack
+         * - HW exception stacks (double fault, nmi, debug, mce)
+         *
+         * x86-32 can have up to three stacks:
+         * - task stack
+         * - softirq stack
+         * - hardirq stack
          */
-        if ((unsigned long)task_stack_page(task) - (unsigned long)stack <
-            PAGE_SIZE)
-                stack = (unsigned long *)task_stack_page(task);
-
-        while (valid_stack_ptr(task, stack, sizeof(*stack), end)) {
-                unsigned long addr;
-
-                addr = *stack;
-                if (__kernel_text_address(addr)) {
-                        if ((unsigned long) stack == bp + sizeof(long)) {
-                                ops->address(data, addr, 1);
-                                frame = frame->next_frame;
-                                bp = (unsigned long) frame;
-                        } else {
-                                ops->address(data, addr, 0);
-                        }
-                        print_ftrace_graph_addr(addr, data, ops, task, graph);
-                }
-                stack++;
-        }
-        return bp;
-}
-EXPORT_SYMBOL_GPL(print_context_stack);
-
-unsigned long
-print_context_stack_bp(struct task_struct *task,
-                       unsigned long *stack, unsigned long bp,
-                       const struct stacktrace_ops *ops, void *data,
-                       unsigned long *end, int *graph)
-{
-        struct stack_frame *frame = (struct stack_frame *)bp;
-        unsigned long *ret_addr = &frame->return_address;
+        for (; stack; stack = stack_info.next_sp) {
+                const char *str_begin, *str_end;
 
-        while (valid_stack_ptr(task, ret_addr, sizeof(*ret_addr), end)) {
-                unsigned long addr = *ret_addr;
+                /*
+                 * If we overflowed the task stack into a guard page, jump back
+                 * to the bottom of the usable stack.
+                 */
+                if (task_stack_page(task) - (void *)stack < PAGE_SIZE)
+                        stack = task_stack_page(task);
 
-                if (!__kernel_text_address(addr))
+                if (get_stack_info(stack, task, &stack_info, &visit_mask))
                         break;
 
-                if (ops->address(data, addr, 1))
-                        break;
-                frame = frame->next_frame;
-                ret_addr = &frame->return_address;
-                print_ftrace_graph_addr(addr, data, ops, task, graph);
-        }
-
-        return (unsigned long)frame;
-}
-EXPORT_SYMBOL_GPL(print_context_stack_bp);
-
-static int print_trace_stack(void *data, char *name)
-{
-        printk("%s <%s> ", (char *)data, name);
-        return 0;
-}
-
-/*
- * Print one address/symbol entries per line.
- */
-static int print_trace_address(void *data, unsigned long addr, int reliable)
-{
-        touch_nmi_watchdog();
-        printk_stack_address(addr, reliable, data);
-        return 0;
-}
-
-static const struct stacktrace_ops print_trace_ops = {
-        .stack                  = print_trace_stack,
-        .address                = print_trace_address,
-        .walk_stack             = print_context_stack,
-};
-
-void
-show_trace_log_lvl(struct task_struct *task, struct pt_regs *regs,
-                unsigned long *stack, unsigned long bp, char *log_lvl)
-{
-        printk("%sCall Trace:\n", log_lvl);
-        dump_trace(task, regs, stack, bp, &print_trace_ops, log_lvl);
-}
+                stack_type_str(stack_info.type, &str_begin, &str_end);
+                if (str_begin)
+                        printk("%s <%s> ", log_lvl, str_begin);
+
+                /*
+                 * Scan the stack, printing any text addresses we find.  At the
+                 * same time, follow proper stack frames with the unwinder.
+                 *
+                 * Addresses found during the scan which are not reported by
+                 * the unwinder are considered to be additional clues which are
+                 * sometimes useful for debugging and are prefixed with '?'.
+                 * This also serves as a failsafe option in case the unwinder
+                 * goes off in the weeds.
+                 */
+                for (; stack < stack_info.end; stack++) {
+                        unsigned long real_addr;
+                        int reliable = 0;
+                        unsigned long addr = *stack;
+                        unsigned long *ret_addr_p =
+                                unwind_get_return_address_ptr(&state);
+
+                        if (!__kernel_text_address(addr))
+                                continue;
+
+                        if (stack == ret_addr_p)
+                                reliable = 1;
+
+                        /*
+                         * When function graph tracing is enabled for a
+                         * function, its return address on the stack is
+                         * replaced with the address of an ftrace handler
+                         * (return_to_handler).  In that case, before printing
+                         * the "real" address, we want to print the handler
+                         * address as an "unreliable" hint that function graph
+                         * tracing was involved.
+                         */
+                        real_addr = ftrace_graph_ret_addr(task, &graph_idx,
+                                                          addr, stack);
+                        if (real_addr != addr)
+                                printk_stack_address(addr, 0, log_lvl);
+                        printk_stack_address(real_addr, reliable, log_lvl);
+
+                        if (!reliable)
+                                continue;
+
+                        /*
+                         * Get the next frame from the unwinder.  No need to
+                         * check for an error: if anything goes wrong, the rest
+                         * of the addresses will just be printed as unreliable.
+                         */
+                        unwind_next_frame(&state);
+                }
 
-void show_trace(struct task_struct *task, struct pt_regs *regs,
-                unsigned long *stack, unsigned long bp)
-{
-        show_trace_log_lvl(task, regs, stack, bp, "");
+                if (str_end)
+                        printk("%s <%s> ", log_lvl, str_end);
+        }
 }
 
 void show_stack(struct task_struct *task, unsigned long *sp)
 {
-        unsigned long bp = 0;
-        unsigned long stack;
+        task = task ? : current;
 
         /*
          * Stack frames below this one aren't interesting.  Don't show them
          * if we're printing for %current.
          */
-        if (!sp && (!task || task == current)) {
-                sp = &stack;
-                bp = stack_frame(current, NULL);
-        }
+        if (!sp && task == current)
+                sp = get_stack_pointer(current, NULL);
 
-        show_stack_log_lvl(task, NULL, sp, bp, "");
+        show_stack_log_lvl(task, NULL, sp, "");
 }
 
 void show_stack_regs(struct pt_regs *regs)
 {
-        show_stack_log_lvl(current, regs, (unsigned long *)regs->sp, regs->bp, "");
+        show_stack_log_lvl(current, regs, NULL, "");
 }
 
 static arch_spinlock_t die_lock = __ARCH_SPIN_LOCK_UNLOCKED;