Merge branch 'x86-asm-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull low-level x86 updates from Ingo Molnar:
 "In this cycle this topic tree has become one of those 'super topics'
  that accumulated a lot of changes:

   - Add CONFIG_VMAP_STACK=y support to the core kernel and enable it on
     x86 - preceded by an array of changes. v4.8 saw preparatory changes
     in this area already - this is the rest of the work. Includes the
     thread stack caching performance optimization. (Andy Lutomirski)

   - switch_to() cleanups and all around enhancements. (Brian Gerst)

   - A large number of dumpstack infrastructure enhancements and an
     unwinder abstraction. The secret long term plan is safe(r) live
     patching plus maybe another attempt at debuginfo based unwinding -
     but all these current bits are standalone enhancements in a frame
     pointer based debug environment as well. (Josh Poimboeuf)

   - More __ro_after_init and const annotations. (Kees Cook)

   - Enable KASLR for the vmemmap memory region. (Thomas Garnier)"

[ The virtually mapped stack changes are pretty fundamental, and not
  x86-specific per se, even if they are only used on x86 right now. ]

* 'x86-asm-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (70 commits)
  x86/asm: Get rid of __read_cr4_safe()
  thread_info: Use unsigned long for flags
  x86/alternatives: Add stack frame dependency to alternative_call_2()
  x86/dumpstack: Fix show_stack() task pointer regression
  x86/dumpstack: Remove dump_trace() and related callbacks
  x86/dumpstack: Convert show_trace_log_lvl() to use the new unwinder
  oprofile/x86: Convert x86_backtrace() to use the new unwinder
  x86/stacktrace: Convert save_stack_trace_*() to use the new unwinder
  perf/x86: Convert perf_callchain_kernel() to use the new unwinder
  x86/unwind: Add new unwind interface and implementations
  x86/dumpstack: Remove NULL task pointer convention
  fork: Optimize task creation by caching two thread stacks per CPU if CONFIG_VMAP_STACK=y
  sched/core: Free the stack early if CONFIG_THREAD_INFO_IN_TASK
  lib/syscall: Pin the task stack in collect_syscall()
  x86/process: Pin the target stack in get_wchan()
  x86/dumpstack: Pin the target stack when dumping it
  kthread: Pin the stack via try_get_task_stack()/put_task_stack() in to_live_kthread() function
  sched/core: Add try_get_task_stack() and put_task_stack()
  x86/entry/64: Fix a minor comment rebase error
  iommu/amd: Don't put completion-wait semaphore on stack
  ...

6 files changed, 233 insertions, 31 deletions

diff --git a/kernel/fork.c b/kernel/fork.c
index beb31725f7e2..c060c7e7c247 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -158,19 +158,83 @@ void __weak arch_release_thread_stack(unsigned long *stack)
  * Allocate pages if THREAD_SIZE is >= PAGE_SIZE, otherwise use a
  * kmemcache based allocator.
  */
-# if THREAD_SIZE >= PAGE_SIZE
-static unsigned long *alloc_thread_stack_node(struct task_struct *tsk,
-                                                  int node)
+# if THREAD_SIZE >= PAGE_SIZE || defined(CONFIG_VMAP_STACK)
+
+#ifdef CONFIG_VMAP_STACK
+/*
+ * vmalloc() is a bit slow, and calling vfree() enough times will force a TLB
+ * flush.  Try to minimize the number of calls by caching stacks.
+ */
+#define NR_CACHED_STACKS 2
+static DEFINE_PER_CPU(struct vm_struct *, cached_stacks[NR_CACHED_STACKS]);
+#endif
+
+static unsigned long *alloc_thread_stack_node(struct task_struct *tsk, int node)
 {
+#ifdef CONFIG_VMAP_STACK
+        void *stack;
+        int i;
+
+        local_irq_disable();
+        for (i = 0; i < NR_CACHED_STACKS; i++) {
+                struct vm_struct *s = this_cpu_read(cached_stacks[i]);
+
+                if (!s)
+                        continue;
+                this_cpu_write(cached_stacks[i], NULL);
+
+                tsk->stack_vm_area = s;
+                local_irq_enable();
+                return s->addr;
+        }
+        local_irq_enable();
+
+        stack = __vmalloc_node_range(THREAD_SIZE, THREAD_SIZE,
+                                     VMALLOC_START, VMALLOC_END,
+                                     THREADINFO_GFP | __GFP_HIGHMEM,
+                                     PAGE_KERNEL,
+                                     0, node, __builtin_return_address(0));
+
+        /*
+         * We can't call find_vm_area() in interrupt context, and
+         * free_thread_stack() can be called in interrupt context,
+         * so cache the vm_struct.
+         */
+        if (stack)
+                tsk->stack_vm_area = find_vm_area(stack);
+        return stack;
+#else
         struct page *page = alloc_pages_node(node, THREADINFO_GFP,
                                              THREAD_SIZE_ORDER);
 
         return page ? page_address(page) : NULL;
+#endif
 }
 
-static inline void free_thread_stack(unsigned long *stack)
+static inline void free_thread_stack(struct task_struct *tsk)
 {
-        __free_pages(virt_to_page(stack), THREAD_SIZE_ORDER);
+#ifdef CONFIG_VMAP_STACK
+        if (task_stack_vm_area(tsk)) {
+                unsigned long flags;
+                int i;
+
+                local_irq_save(flags);
+                for (i = 0; i < NR_CACHED_STACKS; i++) {
+                        if (this_cpu_read(cached_stacks[i]))
+                                continue;
+
+                        this_cpu_write(cached_stacks[i], tsk->stack_vm_area);
+                        local_irq_restore(flags);
+                        return;
+                }
+                local_irq_restore(flags);
+
+                vfree(tsk->stack);
+                return;
+        }
+#endif
+
+        __free_pages(virt_to_page(tsk->stack), THREAD_SIZE_ORDER);
 }
 # else
 static struct kmem_cache *thread_stack_cache;
@@ -181,9 +245,9 @@ static unsigned long *alloc_thread_stack_node(struct task_struct *tsk,
         return kmem_cache_alloc_node(thread_stack_cache, THREADINFO_GFP, node);
 }
 
-static void free_thread_stack(unsigned long *stack)
+static void free_thread_stack(struct task_struct *tsk)
 {
-        kmem_cache_free(thread_stack_cache, stack);
+        kmem_cache_free(thread_stack_cache, tsk->stack);
 }
 
 void thread_stack_cache_init(void)
@@ -213,24 +277,76 @@ struct kmem_cache *vm_area_cachep;
 /* SLAB cache for mm_struct structures (tsk->mm) */
 static struct kmem_cache *mm_cachep;
 
-static void account_kernel_stack(unsigned long *stack, int account)
+static void account_kernel_stack(struct task_struct *tsk, int account)
 {
-        /* All stack pages are in the same zone and belong to the same memcg. */
-        struct page *first_page = virt_to_page(stack);
+        void *stack = task_stack_page(tsk);
+        struct vm_struct *vm = task_stack_vm_area(tsk);
+
+        BUILD_BUG_ON(IS_ENABLED(CONFIG_VMAP_STACK) && PAGE_SIZE % 1024 != 0);
+
+        if (vm) {
+                int i;
 
-        mod_zone_page_state(page_zone(first_page), NR_KERNEL_STACK_KB,
-                            THREAD_SIZE / 1024 * account);
+                BUG_ON(vm->nr_pages != THREAD_SIZE / PAGE_SIZE);
 
-        memcg_kmem_update_page_stat(
-                first_page, MEMCG_KERNEL_STACK_KB,
-                account * (THREAD_SIZE / 1024));
+                for (i = 0; i < THREAD_SIZE / PAGE_SIZE; i++) {
+                        mod_zone_page_state(page_zone(vm->pages[i]),
+                                            NR_KERNEL_STACK_KB,
+                                            PAGE_SIZE / 1024 * account);
+                }
+
+                /* All stack pages belong to the same memcg. */
+                memcg_kmem_update_page_stat(vm->pages[0], MEMCG_KERNEL_STACK_KB,
+                                            account * (THREAD_SIZE / 1024));
+        } else {
+                /*
+                 * All stack pages are in the same zone and belong to the
+                 * same memcg.
+                 */
+                struct page *first_page = virt_to_page(stack);
+
+                mod_zone_page_state(page_zone(first_page), NR_KERNEL_STACK_KB,
+                                    THREAD_SIZE / 1024 * account);
+
+                memcg_kmem_update_page_stat(first_page, MEMCG_KERNEL_STACK_KB,
+                                            account * (THREAD_SIZE / 1024));
+        }
 }
 
-void free_task(struct task_struct *tsk)
+static void release_task_stack(struct task_struct *tsk)
 {
-        account_kernel_stack(tsk->stack, -1);
+        account_kernel_stack(tsk, -1);
         arch_release_thread_stack(tsk->stack);
-        free_thread_stack(tsk->stack);
+        free_thread_stack(tsk);
+        tsk->stack = NULL;
+#ifdef CONFIG_VMAP_STACK
+        tsk->stack_vm_area = NULL;
+#endif
+}
+
+#ifdef CONFIG_THREAD_INFO_IN_TASK
+void put_task_stack(struct task_struct *tsk)
+{
+        if (atomic_dec_and_test(&tsk->stack_refcount))
+                release_task_stack(tsk);
+}
+#endif
+
+void free_task(struct task_struct *tsk)
+{
+#ifndef CONFIG_THREAD_INFO_IN_TASK
+        /*
+         * The task is finally done with both the stack and thread_info,
+         * so free both.
+         */
+        release_task_stack(tsk);
+#else
+        /*
+         * If the task had a separate stack allocation, it should be gone
+         * by now.
+         */
+        WARN_ON_ONCE(atomic_read(&tsk->stack_refcount) != 0);
+#endif
         rt_mutex_debug_task_free(tsk);
         ftrace_graph_exit_task(tsk);
         put_seccomp_filter(tsk);
@@ -342,6 +458,7 @@ static struct task_struct *dup_task_struct(struct task_struct *orig, int node)
 {
         struct task_struct *tsk;
         unsigned long *stack;
+        struct vm_struct *stack_vm_area;
         int err;
 
         if (node == NUMA_NO_NODE)
@@ -354,11 +471,26 @@ static struct task_struct *dup_task_struct(struct task_struct *orig, int node)
         if (!stack)
                 goto free_tsk;
 
+        stack_vm_area = task_stack_vm_area(tsk);
+
         err = arch_dup_task_struct(tsk, orig);
+
+        /*
+         * arch_dup_task_struct() clobbers the stack-related fields.  Make
+         * sure they're properly initialized before using any stack-related
+         * functions again.
+         */
+        tsk->stack = stack;
+#ifdef CONFIG_VMAP_STACK
+        tsk->stack_vm_area = stack_vm_area;
+#endif
+#ifdef CONFIG_THREAD_INFO_IN_TASK
+        atomic_set(&tsk->stack_refcount, 1);
+#endif
+
         if (err)
                 goto free_stack;
 
-        tsk->stack = stack;
 #ifdef CONFIG_SECCOMP
         /*
          * We must handle setting up seccomp filters once we're under
@@ -390,14 +522,14 @@ static struct task_struct *dup_task_struct(struct task_struct *orig, int node)
         tsk->task_frag.page = NULL;
         tsk->wake_q.next = NULL;
 
-        account_kernel_stack(stack, 1);
+        account_kernel_stack(tsk, 1);
 
         kcov_task_init(tsk);
 
         return tsk;
 
 free_stack:
-        free_thread_stack(stack);
+        free_thread_stack(tsk);
 free_tsk:
         free_task_struct(tsk);
         return NULL;
@@ -1715,6 +1847,7 @@ bad_fork_cleanup_count:
         atomic_dec(&p->cred->user->processes);
         exit_creds(p);
 bad_fork_free:
+        put_task_stack(p);
         free_task(p);
 fork_out:
         return ERR_PTR(retval);
diff --git a/kernel/kthread.c b/kernel/kthread.c
index 9ff173dca1ae..4ab4c3766a80 100644
--- a/kernel/kthread.c
+++ b/kernel/kthread.c
@@ -64,7 +64,7 @@ static inline struct kthread *to_kthread(struct task_struct *k)
 static struct kthread *to_live_kthread(struct task_struct *k)
 {
         struct completion *vfork = ACCESS_ONCE(k->vfork_done);
-        if (likely(vfork))
+        if (likely(vfork) && try_get_task_stack(k))
                 return __to_kthread(vfork);
         return NULL;
 }
@@ -425,8 +425,10 @@ void kthread_unpark(struct task_struct *k)
 {
         struct kthread *kthread = to_live_kthread(k);
 
-        if (kthread)
+        if (kthread) {
                 __kthread_unpark(k, kthread);
+                put_task_stack(k);
+        }
 }
 EXPORT_SYMBOL_GPL(kthread_unpark);
 
@@ -455,6 +457,7 @@ int kthread_park(struct task_struct *k)
                                 wait_for_completion(&kthread->parked);
                         }
                 }
+                put_task_stack(k);
                 ret = 0;
         }
         return ret;
@@ -490,6 +493,7 @@ int kthread_stop(struct task_struct *k)
                 __kthread_unpark(k, kthread);
                 wake_up_process(k);
                 wait_for_completion(&kthread->exited);
+                put_task_stack(k);
         }
         ret = k->exit_code;
         put_task_struct(k);
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index fac6492f0b98..94732d1ab00a 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -2781,6 +2781,10 @@ static struct rq *finish_task_switch(struct task_struct *prev)
                  * task and put them back on the free list.
                  */
                 kprobe_flush_task(prev);
+
+                /* Task is done with its stack. */
+                put_task_stack(prev);
+
                 put_task_struct(prev);
         }
 
@@ -3403,7 +3407,6 @@ static void __sched notrace __schedule(bool preempt)
 
         balance_callback(rq);
 }
-STACK_FRAME_NON_STANDARD(__schedule); /* switch_to() */
 
 void __noreturn do_task_dead(void)
 {
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 58df5590d028..055f935d4421 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -1021,7 +1021,11 @@ static inline void __set_task_cpu(struct task_struct *p, unsigned int cpu)
          * per-task data have been completed by this moment.
          */
         smp_wmb();
+#ifdef CONFIG_THREAD_INFO_IN_TASK
+        p->cpu = cpu;
+#else
         task_thread_info(p)->cpu = cpu;
+#endif
         p->wake_cpu = cpu;
 #endif
 }
diff --git a/kernel/trace/Kconfig b/kernel/trace/Kconfig
index f4b86e8ca1e7..ba3326785ca4 100644
--- a/kernel/trace/Kconfig
+++ b/kernel/trace/Kconfig
@@ -24,11 +24,6 @@ config HAVE_FUNCTION_GRAPH_TRACER
         help
           See Documentation/trace/ftrace-design.txt
 
-config HAVE_FUNCTION_GRAPH_FP_TEST
-        bool
-        help
-          See Documentation/trace/ftrace-design.txt
-
 config HAVE_DYNAMIC_FTRACE
         bool
         help
diff --git a/kernel/trace/trace_functions_graph.c b/kernel/trace/trace_functions_graph.c
index 7363ccf79512..0cbe38a844fa 100644
--- a/kernel/trace/trace_functions_graph.c
+++ b/kernel/trace/trace_functions_graph.c
@@ -119,7 +119,7 @@ print_graph_duration(struct trace_array *tr, unsigned long long duration,
 /* Add a function return address to the trace stack on thread info.*/
 int
 ftrace_push_return_trace(unsigned long ret, unsigned long func, int *depth,
-                         unsigned long frame_pointer)
+                         unsigned long frame_pointer, unsigned long *retp)
 {
         unsigned long long calltime;
         int index;
@@ -171,7 +171,12 @@ ftrace_push_return_trace(unsigned long ret, unsigned long func, int *depth,
         current->ret_stack[index].func = func;
         current->ret_stack[index].calltime = calltime;
         current->ret_stack[index].subtime = 0;
+#ifdef HAVE_FUNCTION_GRAPH_FP_TEST
         current->ret_stack[index].fp = frame_pointer;
+#endif
+#ifdef HAVE_FUNCTION_GRAPH_RET_ADDR_PTR
+        current->ret_stack[index].retp = retp;
+#endif
         *depth = current->curr_ret_stack;
 
         return 0;
@@ -204,7 +209,7 @@ ftrace_pop_return_trace(struct ftrace_graph_ret *trace, unsigned long *ret,
                 return;
         }
 
-#if defined(CONFIG_HAVE_FUNCTION_GRAPH_FP_TEST) && !defined(CC_USING_FENTRY)
+#ifdef HAVE_FUNCTION_GRAPH_FP_TEST
         /*
          * The arch may choose to record the frame pointer used
          * and check it here to make sure that it is what we expect it
@@ -279,6 +284,64 @@ unsigned long ftrace_return_to_handler(unsigned long frame_pointer)
         return ret;
 }
 
+/**
+ * ftrace_graph_ret_addr - convert a potentially modified stack return address
+ *                         to its original value
+ *
+ * This function can be called by stack unwinding code to convert a found stack
+ * return address ('ret') to its original value, in case the function graph
+ * tracer has modified it to be 'return_to_handler'.  If the address hasn't
+ * been modified, the unchanged value of 'ret' is returned.
+ *
+ * 'idx' is a state variable which should be initialized by the caller to zero
+ * before the first call.
+ *
+ * 'retp' is a pointer to the return address on the stack.  It's ignored if
+ * the arch doesn't have HAVE_FUNCTION_GRAPH_RET_ADDR_PTR defined.
+ */
+#ifdef HAVE_FUNCTION_GRAPH_RET_ADDR_PTR
+unsigned long ftrace_graph_ret_addr(struct task_struct *task, int *idx,
+                                    unsigned long ret, unsigned long *retp)
+{
+        int index = task->curr_ret_stack;
+        int i;
+
+        if (ret != (unsigned long)return_to_handler)
+                return ret;
+
+        if (index < -1)
+                index += FTRACE_NOTRACE_DEPTH;
+
+        if (index < 0)
+                return ret;
+
+        for (i = 0; i <= index; i++)
+                if (task->ret_stack[i].retp == retp)
+                        return task->ret_stack[i].ret;
+
+        return ret;
+}
+#else /* !HAVE_FUNCTION_GRAPH_RET_ADDR_PTR */
+unsigned long ftrace_graph_ret_addr(struct task_struct *task, int *idx,
+                                    unsigned long ret, unsigned long *retp)
+{
+        int task_idx;
+
+        if (ret != (unsigned long)return_to_handler)
+                return ret;
+
+        task_idx = task->curr_ret_stack;
+
+        if (!task->ret_stack || task_idx < *idx)
+                return ret;
+
+        task_idx -= *idx;
+        (*idx)++;
+
+        return task->ret_stack[task_idx].ret;
+}
+#endif /* HAVE_FUNCTION_GRAPH_RET_ADDR_PTR */
+
 int __trace_graph_entry(struct trace_array *tr,
                                 struct ftrace_graph_ent *trace,
                                 unsigned long flags,