MN10300: Make the FPU operate in non-lazy mode under SMP

Make the FPU operate in non-lazy mode under SMP so that when the process that
is currently using the FPU migrates to a different CPU, we don't have to ping
its previous CPU to flush the FPU context.

Signed-off-by: Akira Takeuchi <takeuchi.akr@jp.panasonic.com>
Signed-off-by: Kiyoshi Owada <owada.kiyoshi@jp.panasonic.com>
Signed-off-by: David Howells <dhowells@redhat.com>

5 files changed, 121 insertions, 56 deletions

diff --git a/arch/mn10300/include/asm/elf.h b/arch/mn10300/include/asm/elf.h
index e5fa97cd9a14..a30d220de5ca 100644
--- a/arch/mn10300/include/asm/elf.h
+++ b/arch/mn10300/include/asm/elf.h
@@ -47,8 +47,6 @@ typedef struct {
         u_int32_t       fpcr;
 } elf_fpregset_t;
 
-extern int dump_fpu(struct pt_regs *, elf_fpregset_t *);
-
 /*
  * This is used to ensure we don't load something for the wrong architecture
  */
diff --git a/arch/mn10300/include/asm/exceptions.h b/arch/mn10300/include/asm/exceptions.h
index 3f3826abc745..7d8080bc6590 100644
--- a/arch/mn10300/include/asm/exceptions.h
+++ b/arch/mn10300/include/asm/exceptions.h
@@ -101,7 +101,6 @@ extern asmlinkage void dtlb_aerror(void);
 extern asmlinkage void raw_bus_error(void);
 extern asmlinkage void double_fault(void);
 extern asmlinkage int  system_call(struct pt_regs *);
-extern asmlinkage void fpu_exception(struct pt_regs *, enum exception_code);
 extern asmlinkage void nmi(struct pt_regs *, enum exception_code);
 extern asmlinkage void uninitialised_exception(struct pt_regs *,
                                                enum exception_code);
diff --git a/arch/mn10300/include/asm/fpu.h b/arch/mn10300/include/asm/fpu.h
index 64a2b83a7a6a..b7625de8eade 100644
--- a/arch/mn10300/include/asm/fpu.h
+++ b/arch/mn10300/include/asm/fpu.h
@@ -12,74 +12,125 @@
 #ifndef _ASM_FPU_H
 #define _ASM_FPU_H
 
-#include <asm/processor.h>
+#ifndef __ASSEMBLY__
+
+#include <linux/sched.h>
+#include <asm/exceptions.h>
 #include <asm/sigcontext.h>
-#include <asm/user.h>
 
 #ifdef __KERNEL__
 
-/* the task that owns the FPU state */
+extern asmlinkage void fpu_disabled(void);
+
+#ifdef CONFIG_FPU
+
+#ifdef CONFIG_LAZY_SAVE_FPU
+/* the task that currently owns the FPU state */
 extern struct task_struct *fpu_state_owner;
+#endif
 
-#define set_using_fpu(tsk)                              \
-do {                                                    \
-        (tsk)->thread.fpu_flags |= THREAD_USING_FPU;    \
-} while (0)
+#if (THREAD_USING_FPU & ~0xff)
+#error THREAD_USING_FPU must be smaller than 0x100.
+#endif
 
-#define clear_using_fpu(tsk)                            \
-do {                                                    \
-        (tsk)->thread.fpu_flags &= ~THREAD_USING_FPU;   \
-} while (0)
+static inline void set_using_fpu(struct task_struct *tsk)
+{
+        asm volatile(
+                "bset %0,(0,%1)"
+                :
+                : "i"(THREAD_USING_FPU), "a"(&tsk->thread.fpu_flags)
+                : "memory", "cc");
+}
 
-#define is_using_fpu(tsk) ((tsk)->thread.fpu_flags & THREAD_USING_FPU)
+static inline void clear_using_fpu(struct task_struct *tsk)
+{
+        asm volatile(
+                "bclr %0,(0,%1)"
+                :
+                : "i"(THREAD_USING_FPU), "a"(&tsk->thread.fpu_flags)
+                : "memory", "cc");
+}
 
-#define unlazy_fpu(tsk)                                 \
-do {                                                    \
-        preempt_disable();                              \
-        if (fpu_state_owner == (tsk))                   \
-                fpu_save(&tsk->thread.fpu_state);       \
-        preempt_enable();                               \
-} while (0)
-
-#define exit_fpu()                              \
-do {                                            \
-        struct task_struct *__tsk = current;    \
-        preempt_disable();                      \
-        if (fpu_state_owner == __tsk)           \
-                fpu_state_owner = NULL;         \
-        preempt_enable();                       \
-} while (0)
-
-#define flush_fpu()                                     \
-do {                                                    \
-        struct task_struct *__tsk = current;            \
-        preempt_disable();                              \
-        if (fpu_state_owner == __tsk) {                 \
-                fpu_state_owner = NULL;                 \
-                __tsk->thread.uregs->epsw &= ~EPSW_FE;  \
-        }                                               \
-        preempt_enable();                               \
-        clear_using_fpu(__tsk);                         \
-} while (0)
+#define is_using_fpu(tsk) ((tsk)->thread.fpu_flags & THREAD_USING_FPU)
 
-extern asmlinkage void fpu_init_state(void);
 extern asmlinkage void fpu_kill_state(struct task_struct *);
-extern asmlinkage void fpu_disabled(struct pt_regs *, enum exception_code);
 extern asmlinkage void fpu_exception(struct pt_regs *, enum exception_code);
-
-#ifdef CONFIG_FPU
+extern asmlinkage void fpu_invalid_op(struct pt_regs *, enum exception_code);
+extern asmlinkage void fpu_init_state(void);
 extern asmlinkage void fpu_save(struct fpu_state_struct *);
-extern asmlinkage void fpu_restore(struct fpu_state_struct *);
-#else
-#define fpu_save(a)
-#define fpu_restore(a)
-#endif /* CONFIG_FPU  */
-
-/*
- * signal frame handlers
- */
 extern int fpu_setup_sigcontext(struct fpucontext *buf);
 extern int fpu_restore_sigcontext(struct fpucontext *buf);
 
+static inline void unlazy_fpu(struct task_struct *tsk)
+{
+        preempt_disable();
+#ifndef CONFIG_LAZY_SAVE_FPU
+        if (tsk->thread.fpu_flags & THREAD_HAS_FPU) {
+                fpu_save(&tsk->thread.fpu_state);
+                tsk->thread.fpu_flags &= ~THREAD_HAS_FPU;
+                tsk->thread.uregs->epsw &= ~EPSW_FE;
+        }
+#else
+        if (fpu_state_owner == tsk)
+                fpu_save(&tsk->thread.fpu_state);
+#endif
+        preempt_enable();
+}
+
+static inline void exit_fpu(void)
+{
+#ifdef CONFIG_LAZY_SAVE_FPU
+        struct task_struct *tsk = current;
+
+        preempt_disable();
+        if (fpu_state_owner == tsk)
+                fpu_state_owner = NULL;
+        preempt_enable();
+#endif
+}
+
+static inline void flush_fpu(void)
+{
+        struct task_struct *tsk = current;
+
+        preempt_disable();
+#ifndef CONFIG_LAZY_SAVE_FPU
+        if (tsk->thread.fpu_flags & THREAD_HAS_FPU) {
+                tsk->thread.fpu_flags &= ~THREAD_HAS_FPU;
+                tsk->thread.uregs->epsw &= ~EPSW_FE;
+        }
+#else
+        if (fpu_state_owner == tsk) {
+                fpu_state_owner = NULL;
+                tsk->thread.uregs->epsw &= ~EPSW_FE;
+        }
+#endif
+        preempt_enable();
+        clear_using_fpu(tsk);
+}
+
+#else /* CONFIG_FPU */
+
+extern asmlinkage
+void unexpected_fpu_exception(struct pt_regs *, enum exception_code);
+#define fpu_invalid_op unexpected_fpu_exception
+#define fpu_exception unexpected_fpu_exception
+
+struct task_struct;
+struct fpu_state_struct;
+static inline bool is_using_fpu(struct task_struct *tsk) { return false; }
+static inline void set_using_fpu(struct task_struct *tsk) {}
+static inline void clear_using_fpu(struct task_struct *tsk) {}
+static inline void fpu_init_state(void) {}
+static inline void fpu_save(struct fpu_state_struct *s) {}
+static inline void fpu_kill_state(struct task_struct *tsk) {}
+static inline void unlazy_fpu(struct task_struct *tsk) {}
+static inline void exit_fpu(void) {}
+static inline void flush_fpu(void) {}
+static inline int fpu_setup_sigcontext(struct fpucontext *buf) { return 0; }
+static inline int fpu_restore_sigcontext(struct fpucontext *buf) { return 0; }
+#endif /* CONFIG_FPU  */
+
 #endif /* __KERNEL__ */
+#endif /* !__ASSEMBLY__ */
 #endif /* _ASM_FPU_H */
diff --git a/arch/mn10300/include/asm/processor.h b/arch/mn10300/include/asm/processor.h
index fd96c180e649..0032fc76c8ba 100644
--- a/arch/mn10300/include/asm/processor.h
+++ b/arch/mn10300/include/asm/processor.h
@@ -95,6 +95,7 @@ struct thread_struct {
         struct pt_regs          *__frame;
         unsigned long           fpu_flags;
 #define THREAD_USING_FPU        0x00000001      /* T if this task is using the FPU */
+#define THREAD_HAS_FPU          0x00000002      /* T if this task owns the FPU right now */
         struct fpu_state_struct fpu_state;
 };
 
diff --git a/arch/mn10300/include/asm/system.h b/arch/mn10300/include/asm/system.h
index 9f7c7e17c01e..3c272a1a1015 100644
--- a/arch/mn10300/include/asm/system.h
+++ b/arch/mn10300/include/asm/system.h
@@ -19,6 +19,21 @@
 #include <linux/kernel.h>
 #include <linux/irqflags.h>
 
+#if !defined(CONFIG_LAZY_SAVE_FPU)
+struct fpu_state_struct;
+extern asmlinkage void fpu_save(struct fpu_state_struct *);
+#define switch_fpu(prev, next)                                          \
+        do {                                                            \
+                if ((prev)->thread.fpu_flags & THREAD_HAS_FPU) {        \
+                        (prev)->thread.fpu_flags &= ~THREAD_HAS_FPU;    \
+                        (prev)->thread.uregs->epsw &= ~EPSW_FE;         \
+                        fpu_save(&(prev)->thread.fpu_state);            \
+                }                                                       \
+        } while (0)
+#else
+#define switch_fpu(prev, next) do {} while (0)
+#endif
+
 struct task_struct;
 struct thread_struct;
 
@@ -30,6 +45,7 @@ struct task_struct *__switch_to(struct thread_struct *prev,
 /* context switching is now performed out-of-line in switch_to.S */
 #define switch_to(prev, next, last)                                     \
 do {                                                                    \
+        switch_fpu(prev, next);                                         \
         current->thread.wchan = (u_long) __builtin_return_address(0);   \
         (last) = __switch_to(&(prev)->thread, &(next)->thread, (prev)); \
         mb();                                                           \