tile: support CONTEXT_TRACKING and thus NOHZ_FULL

Add the TIF_NOHZ flag appropriately.

Add call to user_exit() on entry to do_work_pending() and on entry
to syscalls via do_syscall_trace_enter(), and also the top of
do_syscall_trace_exit() just because it's done in x86.

Add call to user_enter() at the bottom of do_work_pending() once we
have no more work to do before returning to userspace.

Wrap all the trap code in exception_enter() / exception_exit().

Signed-off-by: Chris Metcalf <cmetcalf@ezchip.com>
Acked-by: Frederic Weisbecker <fweisbec@gmail.com>

8 files changed, 67 insertions, 28 deletions

diff --git a/arch/tile/Kconfig b/arch/tile/Kconfig
index 7cca41842a9e..c3a31f8bb09c 100644
--- a/arch/tile/Kconfig
+++ b/arch/tile/Kconfig
@@ -27,6 +27,7 @@ config TILE
         select ARCH_HAS_ATOMIC64_DEC_IF_POSITIVE
         select HAVE_DEBUG_STACKOVERFLOW
         select ARCH_WANT_FRAME_POINTERS
+        select HAVE_CONTEXT_TRACKING
 
 # FIXME: investigate whether we need/want these options.
 #       select HAVE_IOREMAP_PROT
diff --git a/arch/tile/include/asm/thread_info.h b/arch/tile/include/asm/thread_info.h
index 96c14c1430d8..6130a3db505b 100644
--- a/arch/tile/include/asm/thread_info.h
+++ b/arch/tile/include/asm/thread_info.h
@@ -126,6 +126,7 @@ extern void _cpu_idle(void);
 #define TIF_NOTIFY_RESUME       8       /* callback before returning to user */
 #define TIF_SYSCALL_TRACEPOINT  9       /* syscall tracepoint instrumentation */
 #define TIF_POLLING_NRFLAG      10      /* idle is polling for TIF_NEED_RESCHED */
+#define TIF_NOHZ                11      /* in adaptive nohz mode */
 
 #define _TIF_SIGPENDING         (1<<TIF_SIGPENDING)
 #define _TIF_NEED_RESCHED       (1<<TIF_NEED_RESCHED)
@@ -138,14 +139,16 @@ extern void _cpu_idle(void);
 #define _TIF_NOTIFY_RESUME      (1<<TIF_NOTIFY_RESUME)
 #define _TIF_SYSCALL_TRACEPOINT (1<<TIF_SYSCALL_TRACEPOINT)
 #define _TIF_POLLING_NRFLAG     (1<<TIF_POLLING_NRFLAG)
+#define _TIF_NOHZ               (1<<TIF_NOHZ)
 
 /* Work to do on any return to user space. */
 #define _TIF_ALLWORK_MASK \
-  (_TIF_SIGPENDING|_TIF_NEED_RESCHED|_TIF_SINGLESTEP|\
-   _TIF_ASYNC_TLB|_TIF_NOTIFY_RESUME)
+        (_TIF_SIGPENDING | _TIF_NEED_RESCHED | _TIF_SINGLESTEP | \
+         _TIF_ASYNC_TLB | _TIF_NOTIFY_RESUME | _TIF_NOHZ)
 
 /* Work to do at syscall entry. */
-#define _TIF_SYSCALL_ENTRY_WORK (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_TRACEPOINT)
+#define _TIF_SYSCALL_ENTRY_WORK \
+        (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_TRACEPOINT | _TIF_NOHZ)
 
 /* Work to do at syscall exit. */
 #define _TIF_SYSCALL_EXIT_WORK (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_TRACEPOINT)
diff --git a/arch/tile/kernel/process.c b/arch/tile/kernel/process.c
index 48e5773dd0b7..b403c2e3e263 100644
--- a/arch/tile/kernel/process.c
+++ b/arch/tile/kernel/process.c
@@ -27,6 +27,7 @@
 #include <linux/kernel.h>
 #include <linux/tracehook.h>
 #include <linux/signal.h>
+#include <linux/context_tracking.h>
 #include <asm/stack.h>
 #include <asm/switch_to.h>
 #include <asm/homecache.h>
@@ -474,6 +475,8 @@ int do_work_pending(struct pt_regs *regs, u32 thread_info_flags)
         if (!user_mode(regs))
                 return 0;
 
+        user_exit();
+
         /* Enable interrupts; they are disabled again on return to caller. */
         local_irq_enable();
 
@@ -496,11 +499,12 @@ int do_work_pending(struct pt_regs *regs, u32 thread_info_flags)
                 tracehook_notify_resume(regs);
                 return 1;
         }
-        if (thread_info_flags & _TIF_SINGLESTEP) {
+        if (thread_info_flags & _TIF_SINGLESTEP)
                 single_step_once(regs);
-                return 0;
-        }
-        panic("work_pending: bad flags %#x\n", thread_info_flags);
+
+        user_enter();
+
+        return 0;
 }
 
 unsigned long get_wchan(struct task_struct *p)
diff --git a/arch/tile/kernel/ptrace.c b/arch/tile/kernel/ptrace.c
index de98c6ddf136..f84eed8243da 100644
--- a/arch/tile/kernel/ptrace.c
+++ b/arch/tile/kernel/ptrace.c
@@ -22,6 +22,7 @@
 #include <linux/regset.h>
 #include <linux/elf.h>
 #include <linux/tracehook.h>
+#include <linux/context_tracking.h>
 #include <asm/traps.h>
 #include <arch/chip.h>
 
@@ -252,12 +253,21 @@ long compat_arch_ptrace(struct task_struct *child, compat_long_t request,
 
 int do_syscall_trace_enter(struct pt_regs *regs)
 {
-        if (test_thread_flag(TIF_SYSCALL_TRACE)) {
+        u32 work = ACCESS_ONCE(current_thread_info()->flags);
+
+        /*
+         * If TIF_NOHZ is set, we are required to call user_exit() before
+         * doing anything that could touch RCU.
+         */
+        if (work & _TIF_NOHZ)
+                user_exit();
+
+        if (work & _TIF_SYSCALL_TRACE) {
                 if (tracehook_report_syscall_entry(regs))
                         regs->regs[TREG_SYSCALL_NR] = -1;
         }
 
-        if (test_thread_flag(TIF_SYSCALL_TRACEPOINT))
+        if (work & _TIF_SYSCALL_TRACEPOINT)
                 trace_sys_enter(regs, regs->regs[TREG_SYSCALL_NR]);
 
         return regs->regs[TREG_SYSCALL_NR];
@@ -268,6 +278,12 @@ void do_syscall_trace_exit(struct pt_regs *regs)
         long errno;
 
         /*
+         * We may come here right after calling schedule_user()
+         * in which case we can be in RCU user mode.
+         */
+        user_exit();
+
+        /*
          * The standard tile calling convention returns the value (or negative
          * errno) in r0, and zero (or positive errno) in r1.
          * It saves a couple of cycles on the hot path to do this work in
@@ -303,5 +319,7 @@ void send_sigtrap(struct task_struct *tsk, struct pt_regs *regs)
 /* Handle synthetic interrupt delivered only by the simulator. */
 void __kprobes do_breakpoint(struct pt_regs* regs, int fault_num)
 {
+        enum ctx_state prev_state = exception_enter();
         send_sigtrap(current, regs);
+        exception_exit(prev_state);
 }
diff --git a/arch/tile/kernel/single_step.c b/arch/tile/kernel/single_step.c
index 862973074bf9..53f7b9def07b 100644
--- a/arch/tile/kernel/single_step.c
+++ b/arch/tile/kernel/single_step.c
@@ -23,6 +23,7 @@
 #include <linux/types.h>
 #include <linux/err.h>
 #include <linux/prctl.h>
+#include <linux/context_tracking.h>
 #include <asm/cacheflush.h>
 #include <asm/traps.h>
 #include <asm/uaccess.h>
@@ -738,6 +739,7 @@ static DEFINE_PER_CPU(unsigned long, ss_saved_pc);
 
 void gx_singlestep_handle(struct pt_regs *regs, int fault_num)
 {
+        enum ctx_state prev_state = exception_enter();
         unsigned long *ss_pc = this_cpu_ptr(&ss_saved_pc);
         struct thread_info *info = (void *)current_thread_info();
         int is_single_step = test_ti_thread_flag(info, TIF_SINGLESTEP);
@@ -754,6 +756,7 @@ void gx_singlestep_handle(struct pt_regs *regs, int fault_num)
                 __insn_mtspr(SPR_SINGLE_STEP_CONTROL_K, control);
                 send_sigtrap(current, regs);
         }
+        exception_exit(prev_state);
 }
 
 
diff --git a/arch/tile/kernel/traps.c b/arch/tile/kernel/traps.c
index bf841ca517bb..312fc134c1cb 100644
--- a/arch/tile/kernel/traps.c
+++ b/arch/tile/kernel/traps.c
@@ -20,6 +20,7 @@
 #include <linux/reboot.h>
 #include <linux/uaccess.h>
 #include <linux/ptrace.h>
+#include <linux/context_tracking.h>
 #include <asm/stack.h>
 #include <asm/traps.h>
 #include <asm/setup.h>
@@ -253,6 +254,7 @@ static int do_bpt(struct pt_regs *regs)
 void __kprobes do_trap(struct pt_regs *regs, int fault_num,
                        unsigned long reason)
 {
+        enum ctx_state prev_state = exception_enter();
         siginfo_t info = { 0 };
         int signo, code;
         unsigned long address = 0;
@@ -261,7 +263,7 @@ void __kprobes do_trap(struct pt_regs *regs, int fault_num,
 
         /* Handle breakpoints, etc. */
         if (is_kernel && fault_num == INT_ILL && do_bpt(regs))
-                return;
+                goto done;
 
         /* Re-enable interrupts, if they were previously enabled. */
         if (!(regs->flags & PT_FLAGS_DISABLE_IRQ))
@@ -275,7 +277,7 @@ void __kprobes do_trap(struct pt_regs *regs, int fault_num,
                 const char *name;
                 char buf[100];
                 if (fixup_exception(regs))  /* ILL_TRANS or UNALIGN_DATA */
-                        return;
+                        goto done;
                 if (fault_num >= 0 &&
                     fault_num < ARRAY_SIZE(int_name) &&
                     int_name[fault_num] != NULL)
@@ -294,7 +296,6 @@ void __kprobes do_trap(struct pt_regs *regs, int fault_num,
                          fault_num, name, regs->pc, buf);
                 show_regs(regs);
                 do_exit(SIGKILL);  /* FIXME: implement i386 die() */
-                return;
         }
 
         switch (fault_num) {
@@ -308,7 +309,6 @@ void __kprobes do_trap(struct pt_regs *regs, int fault_num,
                         pr_err("Unreadable instruction for INT_ILL: %#lx\n",
                                regs->pc);
                         do_exit(SIGKILL);
-                        return;
                 }
                 if (!special_ill(instr, &signo, &code)) {
                         signo = SIGILL;
@@ -319,7 +319,7 @@ void __kprobes do_trap(struct pt_regs *regs, int fault_num,
         case INT_GPV:
 #if CHIP_HAS_TILE_DMA()
                 if (retry_gpv(reason))
-                        return;
+                        goto done;
 #endif
                 /*FALLTHROUGH*/
         case INT_UDN_ACCESS:
@@ -346,7 +346,7 @@ void __kprobes do_trap(struct pt_regs *regs, int fault_num,
                         if (!state ||
                             (void __user *)(regs->pc) != state->buffer) {
                                 single_step_once(regs);
-                                return;
+                                goto done;
                         }
                 }
 #endif
@@ -380,7 +380,6 @@ void __kprobes do_trap(struct pt_regs *regs, int fault_num,
 #endif
         default:
                 panic("Unexpected do_trap interrupt number %d", fault_num);
-                return;
         }
 
         info.si_signo = signo;
@@ -391,6 +390,9 @@ void __kprobes do_trap(struct pt_regs *regs, int fault_num,
         if (signo != SIGTRAP)
                 trace_unhandled_signal("trap", regs, address, signo);
         force_sig_info(signo, &info, current);
+
+done:
+        exception_exit(prev_state);
 }
 
 void kernel_double_fault(int dummy, ulong pc, ulong lr, ulong sp, ulong r52)
diff --git a/arch/tile/kernel/unaligned.c b/arch/tile/kernel/unaligned.c
index 7d9a83be0aca..d075f92ccee0 100644
--- a/arch/tile/kernel/unaligned.c
+++ b/arch/tile/kernel/unaligned.c
@@ -25,6 +25,7 @@
 #include <linux/module.h>
 #include <linux/compat.h>
 #include <linux/prctl.h>
+#include <linux/context_tracking.h>
 #include <asm/cacheflush.h>
 #include <asm/traps.h>
 #include <asm/uaccess.h>
@@ -1448,6 +1449,7 @@ void jit_bundle_gen(struct pt_regs *regs, tilegx_bundle_bits bundle,
 
 void do_unaligned(struct pt_regs *regs, int vecnum)
 {
+        enum ctx_state prev_state = exception_enter();
         tilegx_bundle_bits __user  *pc;
         tilegx_bundle_bits bundle;
         struct thread_info *info = current_thread_info();
@@ -1487,12 +1489,11 @@ void do_unaligned(struct pt_regs *regs, int vecnum)
                                                 (int)unaligned_fixup,
                                                 (unsigned long long)regs->ex1,
                                                 (unsigned long long)regs->pc);
-                                return;
+                        } else {
+                                /* Not fixable. Go panic. */
+                                panic("Unalign exception in Kernel. pc=%lx",
+                                      regs->pc);
                         }
-                        /* Not fixable. Go panic. */
-                        panic("Unalign exception in Kernel. pc=%lx",
-                              regs->pc);
-                        return;
                 } else {
                         /*
                          * Try to fix the exception. If we can't, panic the
@@ -1501,8 +1502,8 @@ void do_unaligned(struct pt_regs *regs, int vecnum)
                         bundle = GX_INSN_BSWAP(
                                 *((tilegx_bundle_bits *)(regs->pc)));
                         jit_bundle_gen(regs, bundle, align_ctl);
-                        return;
                 }
+                goto done;
         }
 
         /*
@@ -1526,7 +1527,7 @@ void do_unaligned(struct pt_regs *regs, int vecnum)
 
                 trace_unhandled_signal("unaligned fixup trap", regs, 0, SIGBUS);
                 force_sig_info(info.si_signo, &info, current);
-                return;
+                goto done;
         }
 
 
@@ -1543,7 +1544,7 @@ void do_unaligned(struct pt_regs *regs, int vecnum)
                 trace_unhandled_signal("segfault in unalign fixup", regs,
                                        (unsigned long)info.si_addr, SIGSEGV);
                 force_sig_info(info.si_signo, &info, current);
-                return;
+                goto done;
         }
 
         if (!info->unalign_jit_base) {
@@ -1578,7 +1579,7 @@ void do_unaligned(struct pt_regs *regs, int vecnum)
 
                 if (IS_ERR((void __force *)user_page)) {
                         pr_err("Out of kernel pages trying do_mmap\n");
-                        return;
+                        goto done;
                 }
 
                 /* Save the address in the thread_info struct */
@@ -1591,6 +1592,9 @@ void do_unaligned(struct pt_regs *regs, int vecnum)
 
         /* Generate unalign JIT */
         jit_bundle_gen(regs, GX_INSN_BSWAP(bundle), align_ctl);
+
+done:
+        exception_exit(prev_state);
 }
 
 #endif /* __tilegx__ */
diff --git a/arch/tile/mm/fault.c b/arch/tile/mm/fault.c
index 0f61a73534e6..e83cc999da02 100644
--- a/arch/tile/mm/fault.c
+++ b/arch/tile/mm/fault.c
@@ -35,6 +35,7 @@
 #include <linux/syscalls.h>
 #include <linux/uaccess.h>
 #include <linux/kdebug.h>
+#include <linux/context_tracking.h>
 
 #include <asm/pgalloc.h>
 #include <asm/sections.h>
@@ -702,6 +703,7 @@ void do_page_fault(struct pt_regs *regs, int fault_num,
                    unsigned long address, unsigned long write)
 {
         int is_page_fault;
+        enum ctx_state prev_state = exception_enter();
 
 #ifdef CONFIG_KPROBES
         /*
@@ -711,7 +713,7 @@ void do_page_fault(struct pt_regs *regs, int fault_num,
          */
         if (notify_die(DIE_PAGE_FAULT, "page fault", regs, -1,
                        regs->faultnum, SIGSEGV) == NOTIFY_STOP)
-                return;
+                goto done;
 #endif
 
 #ifdef __tilegx__
@@ -750,7 +752,6 @@ void do_page_fault(struct pt_regs *regs, int fault_num,
                                  current->comm, current->pid, pc, address);
                         show_regs(regs);
                         do_group_exit(SIGKILL);
-                        return;
                 }
         }
 #else
@@ -834,12 +835,15 @@ void do_page_fault(struct pt_regs *regs, int fault_num,
                         async->is_fault = is_page_fault;
                         async->is_write = write;
                         async->address = address;
-                        return;
+                        goto done;
                 }
         }
 #endif
 
         handle_page_fault(regs, fault_num, is_page_fault, address, write);
+
+done:
+        exception_exit(prev_state);
 }