powerpc: Better scheduling of CR save code in system call path

At the moment system call entry looks like:

crclr	so
...
mfcr	r9
...
std	r9,_CCR(r1)

commit bd19c8994a82 ([POWERPC] system call micro optimisation) put
some space between the crclr and mfcr in order to avoid a stall.

There is still a stall seen between the mfcr and std. We can avoid
the crclr by doing it in a GPR with rlwinm which gives us more room
to better schedule the sequence.

Signed-off-by: Anton Blanchard <anton@samba.org>
Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>

1 files changed, 7 insertions, 9 deletions

diff --git a/arch/powerpc/kernel/entry_64.S b/arch/powerpc/kernel/entry_64.S
index ba72ddbe5238..fd4604674468 100644
--- a/arch/powerpc/kernel/entry_64.S
+++ b/arch/powerpc/kernel/entry_64.S
@@ -63,15 +63,9 @@ system_call_common:
         std     r0,GPR0(r1)
         std     r10,GPR1(r1)
         ACCOUNT_CPU_USER_ENTRY(r10, r11)
-        /*
-         * This "crclr so" clears CR0.SO, which is the error indication on
-         * return from this system call.  There must be no cmp instruction
-         * between it and the "mfcr r9" below, otherwise if XER.SO is set,
-         * CR0.SO will get set, causing all system calls to appear to fail.
-         */
-        crclr   so
         std     r2,GPR2(r1)
         std     r3,GPR3(r1)
+        mfcr    r2
         std     r4,GPR4(r1)
         std     r5,GPR5(r1)
         std     r6,GPR6(r1)
@@ -85,13 +79,17 @@ system_call_common:
         std     r11,_XER(r1)
         std     r11,_CTR(r1)
         std     r9,GPR13(r1)
-        mfcr    r9
         mflr    r10
+        /*
+         * This clears CR0.SO (bit 28), which is the error indication on
+         * return from this system call.
+         */
+        rldimi  r2,r11,28,(63-28)
         li      r11,0xc01
-        std     r9,_CCR(r1)
         std     r10,_LINK(r1)
         std     r11,_TRAP(r1)
         std     r3,ORIG_GPR3(r1)
+        std     r2,_CCR(r1)
         ld      r2,PACATOC(r13)
         addi    r9,r1,STACK_FRAME_OVERHEAD
         ld      r11,exception_marker@toc(r2)