5 files changed, 238 insertions, 142 deletions

diff --git a/arch/xtensa/include/asm/regs.h b/arch/xtensa/include/asm/regs.h
index b24de6717020..4ba9f516b0e2 100644
--- a/arch/xtensa/include/asm/regs.h
+++ b/arch/xtensa/include/asm/regs.h
@@ -82,6 +82,7 @@
 #define PS_CALLINC_SHIFT        16
 #define PS_CALLINC_MASK         0x00030000
 #define PS_OWB_SHIFT            8
+#define PS_OWB_WIDTH            4
 #define PS_OWB_MASK             0x00000F00
 #define PS_RING_SHIFT           6
 #define PS_RING_MASK            0x000000C0
diff --git a/arch/xtensa/kernel/align.S b/arch/xtensa/kernel/align.S
index aa2e87b8566a..d4cef6039a5c 100644
--- a/arch/xtensa/kernel/align.S
+++ b/arch/xtensa/kernel/align.S
@@ -146,9 +146,9 @@
  *   a0:        trashed, original value saved on stack (PT_AREG0)
  *   a1:        a1
  *   a2:        new stack pointer, original in DEPC
- *   a3:        dispatch table
+ *   a3:        a3
  *   depc:      a2, original value saved on stack (PT_DEPC)
- *   excsave_1: a3
+ *   excsave_1: dispatch table
  *
  *   PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC
  *           <  VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception
@@ -171,7 +171,6 @@ ENTRY(fast_unaligned)
         s32i    a8, a2, PT_AREG8
 
         rsr     a0, depc
-        xsr     a3, excsave1
         s32i    a0, a2, PT_AREG2
         s32i    a3, a2, PT_AREG3
 
diff --git a/arch/xtensa/kernel/coprocessor.S b/arch/xtensa/kernel/coprocessor.S
index 647657484866..a482df5df2b2 100644
--- a/arch/xtensa/kernel/coprocessor.S
+++ b/arch/xtensa/kernel/coprocessor.S
@@ -32,9 +32,9 @@
  *   a0:        trashed, original value saved on stack (PT_AREG0)
  *   a1:        a1
  *   a2:        new stack pointer, original in DEPC
- *   a3:        dispatch table
+ *   a3:        a3
  *   depc:      a2, original value saved on stack (PT_DEPC)
- *   excsave_1: a3
+ *   excsave_1: dispatch table
  *
  *   PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC
  *           <  VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception
@@ -225,9 +225,9 @@ ENDPROC(coprocessor_restore)
  *   a0:        trashed, original value saved on stack (PT_AREG0)
  *   a1:        a1
  *   a2:        new stack pointer, original in DEPC
- *   a3:        dispatch table
+ *   a3:        a3
  *   depc:      a2, original value saved on stack (PT_DEPC)
- *   excsave_1: a3
+ *   excsave_1: dispatch table
  *
  *   PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC
  *           <  VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception
@@ -245,7 +245,6 @@ ENTRY(fast_coprocessor)
 
         /* Save remaining registers a1-a3 and SAR */
 
-        xsr     a3, excsave1
         s32i    a3, a2, PT_AREG3
         rsr     a3, sar
         s32i    a1, a2, PT_AREG1
diff --git a/arch/xtensa/kernel/entry.S b/arch/xtensa/kernel/entry.S
index 3f3de283d707..ab025c1f6e23 100644
--- a/arch/xtensa/kernel/entry.S
+++ b/arch/xtensa/kernel/entry.S
@@ -91,9 +91,9 @@
  *   a0:        trashed, original value saved on stack (PT_AREG0)
  *   a1:        a1
  *   a2:        new stack pointer, original value in depc
- *   a3:        dispatch table
+ *   a3:        a3
  *   depc:      a2, original value saved on stack (PT_DEPC)
- *   excsave1:  a3
+ *   excsave1:  dispatch table
  *
  *   PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC
  *           <  VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception
@@ -109,9 +109,8 @@
 
 ENTRY(user_exception)
 
-        /* Save a2, a3, and depc, restore excsave_1 and set SP. */
+        /* Save a1, a2, a3, and set SP. */
 
-        xsr     a3, excsave1
         rsr     a0, depc
         s32i    a1, a2, PT_AREG1
         s32i    a0, a2, PT_AREG2
@@ -237,9 +236,9 @@ ENDPROC(user_exception)
  *   a0:        trashed, original value saved on stack (PT_AREG0)
  *   a1:        a1
  *   a2:        new stack pointer, original in DEPC
- *   a3:        dispatch table
+ *   a3:        a3
  *   depc:      a2, original value saved on stack (PT_DEPC)
- *   excsave_1: a3
+ *   excsave_1: dispatch table
  *
  *   PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC
  *           <  VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception
@@ -255,9 +254,8 @@ ENDPROC(user_exception)
 
 ENTRY(kernel_exception)
 
-        /* Save a0, a2, a3, DEPC and set SP. */
+        /* Save a1, a2, a3, and set SP. */
 
-        xsr     a3, excsave1            # restore a3, excsave_1
         rsr     a0, depc                # get a2
         s32i    a1, a2, PT_AREG1
         s32i    a0, a2, PT_AREG2
@@ -408,7 +406,7 @@ common_exception:
          * exception handler and call the exception handler.
          */
 
-        movi    a4, exc_table
+        rsr     a4, excsave1
         mov     a6, a1                  # pass stack frame
         mov     a7, a0                  # pass EXCCAUSE
         addx4   a4, a0, a4
@@ -832,9 +830,9 @@ ENDPROC(unrecoverable_exception)
  *   a0:        trashed, original value saved on stack (PT_AREG0)
  *   a1:        a1
  *   a2:        new stack pointer, original in DEPC
- *   a3:        dispatch table
+ *   a3:        a3
  *   depc:      a2, original value saved on stack (PT_DEPC)
- *   excsave_1: a3
+ *   excsave_1: dispatch table
  *
  *   PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC
  *           <  VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception
@@ -857,18 +855,16 @@ ENTRY(fast_alloca)
 
         rsr     a0, depc                # get a2
         s32i    a4, a2, PT_AREG4        # save a4 and
+        s32i    a3, a2, PT_AREG3
         s32i    a0, a2, PT_AREG2        # a2 to stack
 
         /* Exit critical section. */
 
         movi    a0, 0
+        rsr     a3, excsave1
         s32i    a0, a3, EXC_TABLE_FIXUP
 
-        /* Restore a3, excsave_1 */
-
-        xsr     a3, excsave1            # make sure excsave_1 is valid for dbl.
         rsr     a4, epc1                # get exception address
-        s32i    a3, a2, PT_AREG3        # save a3 to stack
 
 #ifdef ALLOCA_EXCEPTION_IN_IRAM
 #error  iram not supported
@@ -1007,9 +1003,9 @@ ENDPROC(fast_alloca)
  *   a0:        trashed, original value saved on stack (PT_AREG0)
  *   a1:        a1
  *   a2:        new stack pointer, original in DEPC
- *   a3:        dispatch table
+ *   a3:        a3
  *   depc:      a2, original value saved on stack (PT_DEPC)
- *   excsave_1: a3
+ *   excsave_1: dispatch table
  */
 
 ENTRY(fast_syscall_kernel)
@@ -1056,7 +1052,6 @@ ENTRY(fast_syscall_unrecoverable)
 
         l32i    a0, a2, PT_AREG0        # restore a0
         xsr     a2, depc                # restore a2, depc
-        rsr     a3, excsave1
 
         wsr     a0, excsave1
         movi    a0, unrecoverable_exception
@@ -1078,10 +1073,10 @@ ENDPROC(fast_syscall_unrecoverable)
  *   a0:        a2 (syscall-nr), original value saved on stack (PT_AREG0)
  *   a1:        a1
  *   a2:        new stack pointer, original in a0 and DEPC
- *   a3:        dispatch table, original in excsave_1
+ *   a3:        a3
  *   a4..a15:   unchanged
  *   depc:      a2, original value saved on stack (PT_DEPC)
- *   excsave_1: a3
+ *   excsave_1: dispatch table
  *
  *   PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC
  *           <  VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception
@@ -1114,8 +1109,6 @@ ENDPROC(fast_syscall_unrecoverable)
 
 ENTRY(fast_syscall_xtensa)
 
-        xsr     a3, excsave1            # restore a3, excsave1
-
         s32i    a7, a2, PT_AREG7        # we need an additional register
         movi    a7, 4                   # sizeof(unsigned int)
         access_ok a3, a7, a0, a2, .Leac # a0: scratch reg, a2: sp
@@ -1178,9 +1171,9 @@ ENDPROC(fast_syscall_xtensa)
  *   a0:        trashed, original value saved on stack (PT_AREG0)
  *   a1:        a1
  *   a2:        new stack pointer, original in DEPC
- *   a3:        dispatch table
+ *   a3:        a3
  *   depc:      a2, original value saved on stack (PT_DEPC)
- *   excsave_1: a3
+ *   excsave_1: dispatch table
  *
  * Note: We assume the stack pointer is EXC_TABLE_KSTK in the fixup handler.
  */
@@ -1189,15 +1182,16 @@ ENTRY(fast_syscall_spill_registers)
 
         /* Register a FIXUP handler (pass current wb as a parameter) */
 
+        xsr     a3, excsave1
         movi    a0, fast_syscall_spill_registers_fixup
         s32i    a0, a3, EXC_TABLE_FIXUP
         rsr     a0, windowbase
         s32i    a0, a3, EXC_TABLE_PARAM
+        xsr     a3, excsave1            # restore a3 and excsave_1
 
-        /* Save a3 and SAR on stack. */
+        /* Save a3, a4 and SAR on stack. */
 
         rsr     a0, sar
-        xsr     a3, excsave1            # restore a3 and excsave_1
         s32i    a3, a2, PT_AREG3
         s32i    a4, a2, PT_AREG4
         s32i    a0, a2, PT_AREG5        # store SAR to PT_AREG5
@@ -1251,14 +1245,14 @@ fast_syscall_spill_registers_fixup:
          * in WS, so that the exception handlers save them to the task stack.
          */
 
-        rsr     a3, excsave1    # get spill-mask
+        xsr     a3, excsave1    # get spill-mask
         slli    a2, a3, 1       # shift left by one
 
         slli    a3, a2, 32-WSBITS
         src     a2, a2, a3      # a1 = xxwww1yyxxxwww1yy......
         wsr     a2, windowstart # set corrected windowstart
 
-        movi    a3, exc_table
+        rsr     a3, excsave1
         l32i    a2, a3, EXC_TABLE_DOUBLE_SAVE   # restore a2
         l32i    a3, a3, EXC_TABLE_PARAM # original WB (in user task)
 
@@ -1295,7 +1289,7 @@ fast_syscall_spill_registers_fixup:
 
         /* Jump to the exception handler. */
 
-        movi    a3, exc_table
+        rsr     a3, excsave1
         rsr     a0, exccause
         addx4   a0, a0, a3                      # find entry in table
         l32i    a0, a0, EXC_TABLE_FAST_USER     # load handler
@@ -1312,6 +1306,7 @@ fast_syscall_spill_registers_fixup_return:
         xsr     a3, excsave1
         movi    a2, fast_syscall_spill_registers_fixup
         s32i    a2, a3, EXC_TABLE_FIXUP
+        s32i    a0, a3, EXC_TABLE_DOUBLE_SAVE
         rsr     a2, windowbase
         s32i    a2, a3, EXC_TABLE_PARAM
         l32i    a2, a3, EXC_TABLE_KSTK
@@ -1323,11 +1318,6 @@ fast_syscall_spill_registers_fixup_return:
         wsr     a3, windowbase
         rsync
 
-        /* Restore a3 and return. */
-
-        movi    a3, exc_table
-        xsr     a3, excsave1
-
         rfde
 
 
@@ -1514,9 +1504,8 @@ ENTRY(_spill_registers)
 
         movi    a0, 0
 
-        movi    a3, exc_table
+        rsr     a3, excsave1
         l32i    a1, a3, EXC_TABLE_KSTK
-        wsr     a3, excsave1
 
         movi    a4, (1 << PS_WOE_BIT) | LOCKLEVEL
         wsr     a4, ps
@@ -1560,9 +1549,9 @@ ENDPROC(fast_second_level_miss_double_kernel)
  *   a0:        trashed, original value saved on stack (PT_AREG0)
  *   a1:        a1
  *   a2:        new stack pointer, original in DEPC
- *   a3:        dispatch table
+ *   a3:        a3
  *   depc:      a2, original value saved on stack (PT_DEPC)
- *   excsave_1: a3
+ *   excsave_1: dispatch table
  *
  *   PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC
  *           <  VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception
@@ -1570,9 +1559,10 @@ ENDPROC(fast_second_level_miss_double_kernel)
 
 ENTRY(fast_second_level_miss)
 
-        /* Save a1. Note: we don't expect a double exception. */
+        /* Save a1 and a3. Note: we don't expect a double exception. */
 
         s32i    a1, a2, PT_AREG1
+        s32i    a3, a2, PT_AREG3
 
         /* We need to map the page of PTEs for the user task.  Find
          * the pointer to that page.  Also, it's possible for tsk->mm
@@ -1594,9 +1584,6 @@ ENTRY(fast_second_level_miss)
         l32i    a0, a1, TASK_MM         # tsk->mm
         beqz    a0, 9f
 
-
-        /* We deliberately destroy a3 that holds the exception table. */
-
 8:      rsr     a3, excvaddr            # fault address
         _PGD_OFFSET(a0, a3, a1)
         l32i    a0, a0, 0               # read pmdval
@@ -1647,7 +1634,7 @@ ENTRY(fast_second_level_miss)
 
         /* Exit critical section. */
 
-4:      movi    a3, exc_table           # restore a3
+4:      rsr     a3, excsave1
         movi    a0, 0
         s32i    a0, a3, EXC_TABLE_FIXUP
 
@@ -1655,8 +1642,8 @@ ENTRY(fast_second_level_miss)
 
         l32i    a0, a2, PT_AREG0
         l32i    a1, a2, PT_AREG1
+        l32i    a3, a2, PT_AREG3
         l32i    a2, a2, PT_DEPC
-        xsr     a3, excsave1
 
         bgeui   a2, VALID_DOUBLE_EXCEPTION_ADDRESS, 1f
 
@@ -1743,11 +1730,8 @@ ENTRY(fast_second_level_miss)
 
 2:      /* Invalid PGD, default exception handling */
 
-        movi    a3, exc_table
         rsr     a1, depc
-        xsr     a3, excsave1
         s32i    a1, a2, PT_AREG2
-        s32i    a3, a2, PT_AREG3
         mov     a1, a2
 
         rsr     a2, ps
@@ -1767,9 +1751,9 @@ ENDPROC(fast_second_level_miss)
  *   a0:        trashed, original value saved on stack (PT_AREG0)
  *   a1:        a1
  *   a2:        new stack pointer, original in DEPC
- *   a3:        dispatch table
+ *   a3:        a3
  *   depc:      a2, original value saved on stack (PT_DEPC)
- *   excsave_1: a3
+ *   excsave_1: dispatch table
  *
  *   PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC
  *           <  VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception
@@ -1777,17 +1761,17 @@ ENDPROC(fast_second_level_miss)
 
 ENTRY(fast_store_prohibited)
 
-        /* Save a1 and a4. */
+        /* Save a1 and a3. */
 
         s32i    a1, a2, PT_AREG1
-        s32i    a4, a2, PT_AREG4
+        s32i    a3, a2, PT_AREG3
 
         GET_CURRENT(a1,a2)
         l32i    a0, a1, TASK_MM         # tsk->mm
         beqz    a0, 9f
 
 8:      rsr     a1, excvaddr            # fault address
-        _PGD_OFFSET(a0, a1, a4)
+        _PGD_OFFSET(a0, a1, a3)
         l32i    a0, a0, 0
         beqz    a0, 2f
 
@@ -1796,39 +1780,37 @@ ENTRY(fast_store_prohibited)
          * and is not PAGE_NONE. See pgtable.h for possible PTE layouts.
          */
 
-        _PTE_OFFSET(a0, a1, a4)
-        l32i    a4, a0, 0               # read pteval
+        _PTE_OFFSET(a0, a1, a3)
+        l32i    a3, a0, 0               # read pteval
         movi    a1, _PAGE_CA_INVALID
-        ball    a4, a1, 2f
-        bbci.l  a4, _PAGE_WRITABLE_BIT, 2f
+        ball    a3, a1, 2f
+        bbci.l  a3, _PAGE_WRITABLE_BIT, 2f
 
         movi    a1, _PAGE_ACCESSED | _PAGE_DIRTY | _PAGE_HW_WRITE
-        or      a4, a4, a1
+        or      a3, a3, a1
         rsr     a1, excvaddr
-        s32i    a4, a0, 0
+        s32i    a3, a0, 0
 
         /* We need to flush the cache if we have page coloring. */
 #if (DCACHE_WAY_SIZE > PAGE_SIZE) && XCHAL_DCACHE_IS_WRITEBACK
         dhwb    a0, 0
 #endif
         pdtlb   a0, a1
-        wdtlb   a4, a0
+        wdtlb   a3, a0
 
         /* Exit critical section. */
 
         movi    a0, 0
+        rsr     a3, excsave1
         s32i    a0, a3, EXC_TABLE_FIXUP
 
         /* Restore the working registers, and return. */
 
-        l32i    a4, a2, PT_AREG4
+        l32i    a3, a2, PT_AREG3
         l32i    a1, a2, PT_AREG1
         l32i    a0, a2, PT_AREG0
         l32i    a2, a2, PT_DEPC
 
-        /* Restore excsave1 and a3. */
-
-        xsr     a3, excsave1
         bgeui   a2, VALID_DOUBLE_EXCEPTION_ADDRESS, 1f
 
         rsr     a2, depc
@@ -1845,11 +1827,8 @@ ENTRY(fast_store_prohibited)
 
 2:      /* If there was a problem, handle fault in C */
 
-        rsr     a4, depc        # still holds a2
-        xsr     a3, excsave1
-        s32i    a4, a2, PT_AREG2
-        s32i    a3, a2, PT_AREG3
-        l32i    a4, a2, PT_AREG4
+        rsr     a3, depc        # still holds a2
+        s32i    a3, a2, PT_AREG2
         mov     a1, a2
 
         rsr     a2, ps
diff --git a/arch/xtensa/kernel/vectors.S b/arch/xtensa/kernel/vectors.S
index f9e175382aa9..cb8fd44caabc 100644
--- a/arch/xtensa/kernel/vectors.S
+++ b/arch/xtensa/kernel/vectors.S
@@ -78,6 +78,7 @@ ENTRY(_UserExceptionVector)
         s32i    a0, a2, PT_DEPC         # mark it as a regular exception
         addx4   a0, a0, a3              # find entry in table
         l32i    a0, a0, EXC_TABLE_FAST_USER     # load handler
+        xsr     a3, excsave1            # restore a3 and dispatch table
         jx      a0
 
 ENDPROC(_UserExceptionVector)
@@ -104,6 +105,7 @@ ENTRY(_KernelExceptionVector)
         s32i    a0, a2, PT_DEPC         # mark it as a regular exception
         addx4   a0, a0, a3              # find entry in table
         l32i    a0, a0, EXC_TABLE_FAST_KERNEL   # load handler address
+        xsr     a3, excsave1            # restore a3 and dispatch table
         jx      a0
 
 ENDPROC(_KernelExceptionVector)
@@ -168,7 +170,7 @@ ENDPROC(_KernelExceptionVector)
  *
  *      a0:        DEPC
  *      a1:        a1
- *      a2:        trashed, original value in EXC_TABLE_DOUBLE_A2
+ *      a2:        trashed, original value in EXC_TABLE_DOUBLE_SAVE
  *      a3:        exctable
  *      depc:      a0
  *      excsave_1: a3
@@ -204,47 +206,46 @@ ENDPROC(_KernelExceptionVector)
 
         .section .DoubleExceptionVector.text, "ax"
         .begin literal_prefix .DoubleExceptionVector
+        .globl _DoubleExceptionVector_WindowUnderflow
+        .globl _DoubleExceptionVector_WindowOverflow
 
 ENTRY(_DoubleExceptionVector)
 
-        /* Deliberately destroy excsave (don't assume it's value was valid). */
-
-        wsr     a3, excsave1            # save a3
+        xsr     a3, excsave1
+        s32i    a2, a3, EXC_TABLE_DOUBLE_SAVE
 
         /* Check for kernel double exception (usually fatal). */
 
-        rsr     a3, ps
-        _bbci.l a3, PS_UM_BIT, .Lksp
+        rsr     a2, ps
+        _bbci.l a2, PS_UM_BIT, .Lksp
 
         /* Check if we are currently handling a window exception. */
         /* Note: We don't need to indicate that we enter a critical section. */
 
         xsr     a0, depc                # get DEPC, save a0
 
-        movi    a3, WINDOW_VECTORS_VADDR
-        _bltu   a0, a3, .Lfixup
-        addi    a3, a3, WINDOW_VECTORS_SIZE
-        _bgeu   a0, a3, .Lfixup
+        movi    a2, WINDOW_VECTORS_VADDR
+        _bltu   a0, a2, .Lfixup
+        addi    a2, a2, WINDOW_VECTORS_SIZE
+        _bgeu   a0, a2, .Lfixup
 
         /* Window overflow/underflow exception. Get stack pointer. */
 
-        mov     a3, a2
-        /* This explicit literal and the following references to it are made
-         * in order to fit DoubleExceptionVector.literals into the available
-         * 16-byte gap before DoubleExceptionVector.text in the absence of
-         * link time relaxation. See kernel/vmlinux.lds.S
-         */
-        .literal .Lexc_table, exc_table
-        l32r    a2, .Lexc_table
-        l32i    a2, a2, EXC_TABLE_KSTK
+        l32i    a2, a3, EXC_TABLE_KSTK
 
         /* Check for overflow/underflow exception, jump if overflow. */
 
-        _bbci.l a0, 6, .Lovfl
-
-        /* a0: depc, a1: a1, a2: kstk, a3: a2, depc: a0, excsave: a3  */
+        _bbci.l a0, 6, _DoubleExceptionVector_WindowOverflow
 
-        /* Restart window underflow exception.
+        /*
+         * Restart window underflow exception.
+         * Currently:
+         *      depc = orig a0,
+         *      a0 = orig DEPC,
+         *      a2 = new sp based on KSTK from exc_table
+         *      a3 = excsave_1
+         *      excsave_1 = orig a3
+         *
          * We return to the instruction in user space that caused the window
          * underflow exception. Therefore, we change window base to the value
          * before we entered the window underflow exception and prepare the
@@ -252,10 +253,11 @@ ENTRY(_DoubleExceptionVector)
          * by changing depc (in a0).
          * Note: We can trash the current window frame (a0...a3) and depc!
          */
-
+_DoubleExceptionVector_WindowUnderflow:
+        xsr     a3, excsave1
         wsr     a2, depc                # save stack pointer temporarily
         rsr     a0, ps
-        extui   a0, a0, PS_OWB_SHIFT, 4
+        extui   a0, a0, PS_OWB_SHIFT, PS_OWB_WIDTH
         wsr     a0, windowbase
         rsync
 
@@ -263,28 +265,57 @@ ENTRY(_DoubleExceptionVector)
 
         xsr     a2, depc                # save a2 and get stack pointer
         s32i    a0, a2, PT_AREG0
-
-        wsr     a3, excsave1            # save a3
-        l32r    a3, .Lexc_table
-
+        xsr     a3, excsave1
         rsr     a0, exccause
         s32i    a0, a2, PT_DEPC         # mark it as a regular exception
         addx4   a0, a0, a3
+        xsr     a3, excsave1
         l32i    a0, a0, EXC_TABLE_FAST_USER
         jx      a0
 
-.Lfixup:/* Check for a fixup handler or if we were in a critical section. */
+        /*
+         * We only allow the ITLB miss exception if we are in kernel space.
+         * All other exceptions are unexpected and thus unrecoverable!
+         */
+
+#ifdef CONFIG_MMU
+        .extern fast_second_level_miss_double_kernel
+
+.Lksp:  /* a0: a0, a1: a1, a2: a2, a3: trashed, depc: depc, excsave: a3 */
+
+        rsr     a3, exccause
+        beqi    a3, EXCCAUSE_ITLB_MISS, 1f
+        addi    a3, a3, -EXCCAUSE_DTLB_MISS
+        bnez    a3, .Lunrecoverable
+1:      movi    a3, fast_second_level_miss_double_kernel
+        jx      a3
+#else
+.equ    .Lksp,  .Lunrecoverable
+#endif
+
+        /* Critical! We can't handle this situation. PANIC! */
 
-        /* a0: depc, a1: a1, a2: a2, a3: trashed, depc: a0, excsave1: a3 */
+        .extern unrecoverable_exception
 
-        l32r    a3, .Lexc_table
-        s32i    a2, a3, EXC_TABLE_DOUBLE_SAVE   # temporary variable
+.Lunrecoverable_fixup:
+        l32i    a2, a3, EXC_TABLE_DOUBLE_SAVE
+        xsr     a0, depc
+
+.Lunrecoverable:
+        rsr     a3, excsave1
+        wsr     a0, excsave1
+        movi    a0, unrecoverable_exception
+        callx0  a0
+
+.Lfixup:/* Check for a fixup handler or if we were in a critical section. */
+
+        /* a0: depc, a1: a1, a2: trash, a3: exctable, depc: a0, excsave1: a3 */
 
         /* Enter critical section. */
 
         l32i    a2, a3, EXC_TABLE_FIXUP
         s32i    a3, a3, EXC_TABLE_FIXUP
-        beq     a2, a3, .Lunrecoverable_fixup   # critical!
+        beq     a2, a3, .Lunrecoverable_fixup   # critical section
         beqz    a2, .Ldflt                      # no handler was registered
 
         /* a0: depc, a1: a1, a2: trash, a3: exctable, depc: a0, excsave: a3 */
@@ -293,58 +324,145 @@ ENTRY(_DoubleExceptionVector)
 
 .Ldflt: /* Get stack pointer. */
 
-        l32i    a3, a3, EXC_TABLE_DOUBLE_SAVE
-        addi    a2, a3, -PT_USER_SIZE
-
-.Lovfl: /* Jump to default handlers. */
+        l32i    a2, a3, EXC_TABLE_DOUBLE_SAVE
+        addi    a2, a2, -PT_USER_SIZE
 
-        /* a0: depc, a1: a1, a2: kstk, a3: a2, depc: a0, excsave: a3 */
+        /* a0: depc, a1: a1, a2: kstk, a3: exctable, depc: a0, excsave: a3 */
 
-        xsr     a3, depc
         s32i    a0, a2, PT_DEPC
-        s32i    a3, a2, PT_AREG0
+        l32i    a0, a3, EXC_TABLE_DOUBLE_SAVE
+        xsr     a0, depc
+        s32i    a0, a2, PT_AREG0
 
-        /* a0: avail, a1: a1, a2: kstk, a3: avail, depc: a2, excsave: a3 */
+        /* a0: avail, a1: a1, a2: kstk, a3: exctable, depc: a2, excsave: a3 */
 
-        l32r    a3, .Lexc_table
         rsr     a0, exccause
         addx4   a0, a0, a3
+        xsr     a3, excsave1
         l32i    a0, a0, EXC_TABLE_FAST_USER
         jx      a0
 
         /*
-         * We only allow the ITLB miss exception if we are in kernel space.
-         * All other exceptions are unexpected and thus unrecoverable!
+         * Restart window OVERFLOW exception.
+         * Currently:
+         *      depc = orig a0,
+         *      a0 = orig DEPC,
+         *      a2 = new sp based on KSTK from exc_table
+         *      a3 = EXCSAVE_1
+         *      excsave_1 = orig a3
+         *
+         * We return to the instruction in user space that caused the window
+         * overflow exception. Therefore, we change window base to the value
+         * before we entered the window overflow exception and prepare the
+         * registers to return as if we were coming from a regular exception
+         * by changing DEPC (in a0).
+         *
+         * NOTE: We CANNOT trash the current window frame (a0...a3), but we
+         * can clobber depc.
+         *
+         * The tricky part here is that overflow8 and overflow12 handlers
+         * save a0, then clobber a0.  To restart the handler, we have to restore
+         * a0 if the double exception was past the point where a0 was clobbered.
+         *
+         * To keep things simple, we take advantage of the fact all overflow
+         * handlers save a0 in their very first instruction.  If DEPC was past
+         * that instruction, we can safely restore a0 from where it was saved
+         * on the stack.
+         *
+         * a0: depc, a1: a1, a2: kstk, a3: exc_table, depc: a0, excsave1: a3
          */
+_DoubleExceptionVector_WindowOverflow:
+        extui   a2, a0, 0, 6    # get offset into 64-byte vector handler
+        beqz    a2, 1f          # if at start of vector, don't restore
 
-#ifdef CONFIG_MMU
-        .extern fast_second_level_miss_double_kernel
+        addi    a0, a0, -128
+        bbsi    a0, 8, 1f       # don't restore except for overflow 8 and 12
+        bbsi    a0, 7, 2f
 
-.Lksp:  /* a0: a0, a1: a1, a2: a2, a3: trashed, depc: depc, excsave: a3 */
+        /*
+         * Restore a0 as saved by _WindowOverflow8().
+         *
+         * FIXME:  we really need a fixup handler for this L32E,
+         * for the extremely unlikely case where the overflow handler's
+         * reference thru a0 gets a hardware TLB refill that bumps out
+         * the (distinct, aliasing) TLB entry that mapped its prior
+         * references thru a9, and where our reference now thru a9
+         * gets a 2nd-level miss exception (not hardware TLB refill).
+         */
 
-        rsr     a3, exccause
-        beqi    a3, EXCCAUSE_ITLB_MISS, 1f
-        addi    a3, a3, -EXCCAUSE_DTLB_MISS
-        bnez    a3, .Lunrecoverable
-1:      movi    a3, fast_second_level_miss_double_kernel
-        jx      a3
-#else
-.equ    .Lksp,  .Lunrecoverable
-#endif
+        l32e    a2, a9, -16
+        wsr     a2, depc        # replace the saved a0
+        j       1f
 
-        /* Critical! We can't handle this situation. PANIC! */
+2:
+        /*
+         * Restore a0 as saved by _WindowOverflow12().
+         *
+         * FIXME:  we really need a fixup handler for this L32E,
+         * for the extremely unlikely case where the overflow handler's
+         * reference thru a0 gets a hardware TLB refill that bumps out
+         * the (distinct, aliasing) TLB entry that mapped its prior
+         * references thru a13, and where our reference now thru a13
+         * gets a 2nd-level miss exception (not hardware TLB refill).
+         */
 
-        .extern unrecoverable_exception
+        l32e    a2, a13, -16
+        wsr     a2, depc        # replace the saved a0
+1:
+        /*
+         * Restore WindowBase while leaving all address registers restored.
+         * We have to use ROTW for this, because WSR.WINDOWBASE requires
+         * an address register (which would prevent restore).
+         *
+         * Window Base goes from 0 ... 7 (Module 8)
+         * Window Start is 8 bits; Ex: (0b1010 1010):0x55 from series of call4s
+         */
+
+        rsr     a0, ps
+        extui   a0, a0, PS_OWB_SHIFT, PS_OWB_WIDTH
+        rsr     a2, windowbase
+        sub     a0, a2, a0
+        extui   a0, a0, 0, 3
 
-.Lunrecoverable_fixup:
         l32i    a2, a3, EXC_TABLE_DOUBLE_SAVE
-        xsr     a0, depc
+        xsr     a3, excsave1
+        beqi    a0, 1, .L1pane
+        beqi    a0, 3, .L3pane
 
-.Lunrecoverable:
-        rsr     a3, excsave1
-        wsr     a0, excsave1
-        movi    a0, unrecoverable_exception
-        callx0  a0
+        rsr     a0, depc
+        rotw    -2
+
+        /*
+         * We are now in the user code's original window frame.
+         * Process the exception as a user exception as if it was
+         * taken by the user code.
+         *
+         * This is similar to the user exception vector,
+         * except that PT_DEPC isn't set to EXCCAUSE.
+         */
+1:
+        xsr     a3, excsave1
+        wsr     a2, depc
+        l32i    a2, a3, EXC_TABLE_KSTK
+        s32i    a0, a2, PT_AREG0
+        rsr     a0, exccause
+
+        s32i    a0, a2, PT_DEPC
+
+        addx4   a0, a0, a3
+        l32i    a0, a0, EXC_TABLE_FAST_USER
+        xsr     a3, excsave1
+        jx      a0
+
+.L1pane:
+        rsr     a0, depc
+        rotw    -1
+        j       1b
+
+.L3pane:
+        rsr     a0, depc
+        rotw    -3
+        j       1b
 
         .end literal_prefix