alpha: fix alignment problem in csum_ipv6_magic()

Hopefully this fixes http://bugzilla.kernel.org/show_bug.cgi?id=8635 The struct in6_addr passed to csum_ipv6_magic() is 4 byte aligned, so we can't use the regular 64-bit loads. Since the cost of handling of 4 byte and 1 byte aligned 64-bit data is roughly the same, this code can cope with any src/dst [mis]alignment. Signed-off-by: Ivan Kokshaysky <ink@jurassic.park.msu.ru> Cc: Richard Henderson <rth@twiddle.net> Cc: Dustin Marquess <jailbird@alcatraz.fdf.net> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
author: Ivan Kokshaysky <ink@jurassic.park.msu.ru> 2007-06-23 20:16:35 -0400
committer: Linus Torvalds <torvalds@woody.linux-foundation.org> 2007-06-24 11:59:11 -0400
commit: 58ed2f9c75b719da4f494f24ed2d56d45f5b4959 (patch)
tree: 6ecb60cf2d7b18da36fd29f2a6bd03d15a8cfefb
parent: 653d4876b730fedca8473481863cf700245e3582 (diff)
2 files changed, 70 insertions, 23 deletions
diff --git a/arch/alpha/lib/csum_ipv6_magic.S b/arch/alpha/lib/csum_ipv6_magic.S
index e09748dbf2ed..2c2acb96deb6 100644
--- a/arch/alpha/lib/csum_ipv6_magic.S
+++ b/arch/alpha/lib/csum_ipv6_magic.S
@@ -7,6 +7,9 @@
 *                                __u32 len,
 *                                unsigned short proto,
 *                                unsigned int csum);
+ *
+ * Misalignment handling (which costs 16 instructions / 8 cycles)
+ * added by Ivan Kokshaysky <ink@jurassic.park.msu.ru>
 */
        .globl csum_ipv6_magic
@@ -16,37 +19,57 @@
 csum_ipv6_magic:
        .prologue 0
-        ldq     $0,0($16)       # e0    : load src & dst addr words
+        ldq_u   $0,0($16)       # e0    : load src & dst addr words
        zapnot  $20,15,$20      # .. e1 : zero extend incoming csum
        extqh   $18,1,$4        # e0    : byte swap len & proto while we wait
-        ldq     $1,8($16)       # .. e1 :
+        ldq_u   $21,7($16)      # .. e1 : handle misalignment
        extbl   $18,1,$5        # e0    :
-        ldq     $2,0($17)       # .. e1 :
+        ldq_u   $1,8($16)       # .. e1 :
        extbl   $18,2,$6        # e0    :
-        ldq     $3,8($17)       # .. e1 :
+        ldq_u   $22,15($16)     # .. e1 :
        extbl   $18,3,$18       # e0    :
+        ldq_u   $2,0($17)       # .. e1 :
        sra     $4,32,$4        # e0    :
+        ldq_u   $23,7($17)      # .. e1 :
+        extql   $0,$16,$0       # e0    :
+        ldq_u   $3,8($17)       # .. e1 :
+        extqh   $21,$16,$21     # e0    :
+        ldq_u   $24,15($17)     # .. e1 :
        sll     $5,16,$5        # e0    :
+        or      $0,$21,$0       # .. e1 : 1st src word complete
+        extql   $1,$16,$1       # e0    :
        addq    $20,$0,$20      # .. e1 : begin summing the words
-        sll     $6,8,$6         # e0    :
+        extqh   $22,$16,$22     # e0    :
        cmpult  $20,$0,$0       # .. e1 :
-        extwh   $19,7,$7        # e0    :
+        sll     $6,8,$6         # e0    :
-        or      $4,$18,$18      # .. e1 :
+        or      $1,$22,$1       # .. e1 : 2nd src word complete
-        extbl   $19,1,$19       # e0    :
+        extql   $2,$17,$2       # e0    :
+        or      $4,$18,$18      # .. e1 :
+        extqh   $23,$17,$23     # e0    :
        or      $5,$6,$5        # .. e1 :
-        or      $18,$5,$18      # e0    : len complete
-        or      $19,$7,$19      # .. e1 :
-        sll     $19,48,$19      # e0    :
+        extql   $3,$17,$3       # e0    :
+        or      $2,$23,$2       # .. e1 : 1st dst word complete
+        extqh   $24,$17,$24     # e0    :
+        or      $18,$5,$18      # .. e1 : len complete
+        extwh   $19,7,$7        # e0    :
+        or      $3,$24,$3       # .. e1 : 2nd dst word complete
+        extbl   $19,1,$19       # e0    :
        addq    $20,$1,$20      # .. e1 :
-        sra     $19,32,$19      # e0    : proto complete
+        or      $19,$7,$19      # e0    :
        cmpult  $20,$1,$1       # .. e1 :
+        sll     $19,48,$19      # e0    :
+        nop                     # .. e0 :
-        nop                     # e0    :
+        sra     $19,32,$19      # e0    : proto complete
        addq    $20,$2,$20      # .. e1 :
        cmpult  $20,$2,$2       # e0    :
        addq    $20,$3,$20      # .. e1 :
@@ -84,7 +107,7 @@ csum_ipv6_magic:
        extwl   $0,2,$1         # e0    : fold 17-bit value
        zapnot  $0,3,$0         # .. e1 :
        addq    $0,$1,$0        # e0    :
-        not     $0,$0           # e1    : and complement.
+        not     $0,$0           # .. e1 : and complement.
        zapnot  $0,3,$0         # e0    :
        ret                     # .. e1 :
diff --git a/arch/alpha/lib/ev6-csum_ipv6_magic.S b/arch/alpha/lib/ev6-csum_ipv6_magic.S
index de1948a69118..fc0bc399f872 100644
--- a/arch/alpha/lib/ev6-csum_ipv6_magic.S
+++ b/arch/alpha/lib/ev6-csum_ipv6_magic.S
@@ -46,6 +46,10 @@
 * add the 3 low ushorts together, generating a uint
 * a final add of the 2 lower ushorts
 * truncating the result.
+ *
+ * Misalignment handling added by Ivan Kokshaysky <ink@jurassic.park.msu.ru>
+ * The cost is 16 instructions (~8 cycles), including two extra loads which
+ * may cause additional delay in rare cases (load-load replay traps).
 */
        .globl csum_ipv6_magic
@@ -55,25 +59,45 @@
 csum_ipv6_magic:
        .prologue 0
-        ldq     $0,0($16)       # L : Latency: 3
+        ldq_u   $0,0($16)       # L : Latency: 3
        inslh   $18,7,$4        # U : 0000000000AABBCC
-        ldq     $1,8($16)       # L : Latency: 3
+        ldq_u   $1,8($16)       # L : Latency: 3
        sll     $19,8,$7        # U : U L U L : 0x00000000 00aabb00
+        and     $16,7,$6        # E : src misalignment
+        ldq_u   $5,15($16)      # L : Latency: 3
        zapnot  $20,15,$20      # U : zero extend incoming csum
-        ldq     $2,0($17)       # L : Latency: 3
+        ldq_u   $2,0($17)       # L : U L U L : Latency: 3
-        sll     $19,24,$19      # U : U L L U : 0x000000aa bb000000
+        extql   $0,$6,$0        # U :
+        extqh   $1,$6,$22       # U :
+        ldq_u   $3,8($17)       # L : Latency: 3
+        sll     $19,24,$19      # U : U U L U : 0x000000aa bb000000
+        cmoveq  $6,$31,$22      # E : src aligned?
+        ldq_u   $23,15($17)     # L : Latency: 3
        inswl   $18,3,$18       # U : 000000CCDD000000
+        addl    $19,$7,$19      # E : U L U L : <sign bits>bbaabb00
-        ldq     $3,8($17)       # L : Latency: 3
+        or      $0,$22,$0       # E : 1st src word complete
-        bis     $18,$4,$18      # E : 000000CCDDAABBCC
+        extql   $1,$6,$1        # U :
-        addl    $19,$7,$19      # E : <sign bits>bbaabb00
+        or      $18,$4,$18      # E : 000000CCDDAABBCC
-        nop                     # E : U L U L
+        extqh   $5,$6,$5        # U : L U L U
+        and     $17,7,$6        # E : dst misalignment
+        extql   $2,$6,$2        # U :
+        or      $1,$5,$1        # E : 2nd src word complete
+        extqh   $3,$6,$22       # U : L U L U :
+        cmoveq  $6,$31,$22      # E : dst aligned?
+        extql   $3,$6,$3        # U :
        addq    $20,$0,$20      # E : begin summing the words
+        extqh   $23,$6,$23      # U : L U L U :
        srl     $18,16,$4       # U : 0000000000CCDDAA
+        or      $2,$22,$2       # E : 1st dst word complete
        zap     $19,0x3,$19     # U : <sign bits>bbaa0000
-        nop                     # E : L U U L
+        or      $3,$23,$3       # E : U L U L : 2nd dst word complete
        cmpult  $20,$0,$0       # E :
        addq    $20,$1,$20      # E :
author	Ivan Kokshaysky <ink@jurassic.park.msu.ru>	2007-06-23 20:16:35 -0400
committer	Linus Torvalds <torvalds@woody.linux-foundation.org>	2007-06-24 11:59:11 -0400
commit	58ed2f9c75b719da4f494f24ed2d56d45f5b4959 (patch)
tree	6ecb60cf2d7b18da36fd29f2a6bd03d15a8cfefb
parent	653d4876b730fedca8473481863cf700245e3582 (diff)

diff --git a/arch/alpha/lib/csum_ipv6_magic.S b/arch/alpha/lib/csum_ipv6_magic.S index e09748dbf2ed..2c2acb96deb6 100644 --- a/arch/alpha/lib/csum_ipv6_magic.S +++ b/arch/alpha/lib/csum_ipv6_magic.S
@@ -7,6 +7,9 @@
7	* __u32 len,	7	* __u32 len,
8	* unsigned short proto,	8	* unsigned short proto,
9	* unsigned int csum);	9	* unsigned int csum);
		10	*
		11	* Misalignment handling (which costs 16 instructions / 8 cycles)
		12	* added by Ivan Kokshaysky <ink@jurassic.park.msu.ru>
10	*/	13	*/
11		14
12	.globl csum_ipv6_magic	15	.globl csum_ipv6_magic
@@ -16,37 +19,57 @@
16	csum_ipv6_magic:	19	csum_ipv6_magic:
17	.prologue 0	20	.prologue 0
18		21
19	ldq $0,0($16) # e0 : load src & dst addr words	22	ldq_u $0,0($16) # e0 : load src & dst addr words
20	zapnot $20,15,$20 # .. e1 : zero extend incoming csum	23	zapnot $20,15,$20 # .. e1 : zero extend incoming csum
21	extqh $18,1,$4 # e0 : byte swap len & proto while we wait	24	extqh $18,1,$4 # e0 : byte swap len & proto while we wait
22	ldq $1,8($16) # .. e1 :	25	ldq_u $21,7($16) # .. e1 : handle misalignment
23		26
24	extbl $18,1,$5 # e0 :	27	extbl $18,1,$5 # e0 :
25	ldq $2,0($17) # .. e1 :	28	ldq_u $1,8($16) # .. e1 :
26	extbl $18,2,$6 # e0 :	29	extbl $18,2,$6 # e0 :
27	ldq $3,8($17) # .. e1 :	30	ldq_u $22,15($16) # .. e1 :
28		31
29	extbl $18,3,$18 # e0 :	32	extbl $18,3,$18 # e0 :
		33	ldq_u $2,0($17) # .. e1 :
30	sra $4,32,$4 # e0 :	34	sra $4,32,$4 # e0 :
		35	ldq_u $23,7($17) # .. e1 :
		36
		37	extql $0,$16,$0 # e0 :
		38	ldq_u $3,8($17) # .. e1 :
		39	extqh $21,$16,$21 # e0 :
		40	ldq_u $24,15($17) # .. e1 :
		41
31	sll $5,16,$5 # e0 :	42	sll $5,16,$5 # e0 :
		43	or $0,$21,$0 # .. e1 : 1st src word complete
		44	extql $1,$16,$1 # e0 :
32	addq $20,$0,$20 # .. e1 : begin summing the words	45	addq $20,$0,$20 # .. e1 : begin summing the words
33		46
34	sll $6,8,$6 # e0 :	47	extqh $22,$16,$22 # e0 :
35	cmpult $20,$0,$0 # .. e1 :	48	cmpult $20,$0,$0 # .. e1 :
36	extwh $19,7,$7 # e0 :	49	sll $6,8,$6 # e0 :
37	or $4,$18,$18 # .. e1 :	50	or $1,$22,$1 # .. e1 : 2nd src word complete
38		51
39	extbl $19,1,$19 # e0 :	52	extql $2,$17,$2 # e0 :
		53	or $4,$18,$18 # .. e1 :
		54	extqh $23,$17,$23 # e0 :
40	or $5,$6,$5 # .. e1 :	55	or $5,$6,$5 # .. e1 :
41	or $18,$5,$18 # e0 : len complete
42	or $19,$7,$19 # .. e1 :
43		56
44	sll $19,48,$19 # e0 :	57	extql $3,$17,$3 # e0 :
		58	or $2,$23,$2 # .. e1 : 1st dst word complete
		59	extqh $24,$17,$24 # e0 :
		60	or $18,$5,$18 # .. e1 : len complete
		61
		62	extwh $19,7,$7 # e0 :
		63	or $3,$24,$3 # .. e1 : 2nd dst word complete
		64	extbl $19,1,$19 # e0 :
45	addq $20,$1,$20 # .. e1 :	65	addq $20,$1,$20 # .. e1 :
46	sra $19,32,$19 # e0 : proto complete	66
		67	or $19,$7,$19 # e0 :
47	cmpult $20,$1,$1 # .. e1 :	68	cmpult $20,$1,$1 # .. e1 :
		69	sll $19,48,$19 # e0 :
		70	nop # .. e0 :
48		71
49	nop # e0 :	72	sra $19,32,$19 # e0 : proto complete
50	addq $20,$2,$20 # .. e1 :	73	addq $20,$2,$20 # .. e1 :
51	cmpult $20,$2,$2 # e0 :	74	cmpult $20,$2,$2 # e0 :
52	addq $20,$3,$20 # .. e1 :	75	addq $20,$3,$20 # .. e1 :
@@ -84,7 +107,7 @@ csum_ipv6_magic:
84	extwl $0,2,$1 # e0 : fold 17-bit value	107	extwl $0,2,$1 # e0 : fold 17-bit value
85	zapnot $0,3,$0 # .. e1 :	108	zapnot $0,3,$0 # .. e1 :
86	addq $0,$1,$0 # e0 :	109	addq $0,$1,$0 # e0 :
87	not $0,$0 # e1 : and complement.	110	not $0,$0 # .. e1 : and complement.
88		111
89	zapnot $0,3,$0 # e0 :	112	zapnot $0,3,$0 # e0 :
90	ret # .. e1 :	113	ret # .. e1 :


diff --git a/arch/alpha/lib/ev6-csum_ipv6_magic.S b/arch/alpha/lib/ev6-csum_ipv6_magic.S index de1948a69118..fc0bc399f872 100644 --- a/arch/alpha/lib/ev6-csum_ipv6_magic.S +++ b/arch/alpha/lib/ev6-csum_ipv6_magic.S
@@ -46,6 +46,10 @@
46	* add the 3 low ushorts together, generating a uint	46	* add the 3 low ushorts together, generating a uint
47	* a final add of the 2 lower ushorts	47	* a final add of the 2 lower ushorts
48	* truncating the result.	48	* truncating the result.
		49	*
		50	* Misalignment handling added by Ivan Kokshaysky <ink@jurassic.park.msu.ru>
		51	* The cost is 16 instructions (~8 cycles), including two extra loads which
		52	* may cause additional delay in rare cases (load-load replay traps).
49	*/	53	*/
50		54
51	.globl csum_ipv6_magic	55	.globl csum_ipv6_magic
@@ -55,25 +59,45 @@
55	csum_ipv6_magic:	59	csum_ipv6_magic:
56	.prologue 0	60	.prologue 0
57		61
58	ldq $0,0($16) # L : Latency: 3	62	ldq_u $0,0($16) # L : Latency: 3
59	inslh $18,7,$4 # U : 0000000000AABBCC	63	inslh $18,7,$4 # U : 0000000000AABBCC
60	ldq $1,8($16) # L : Latency: 3	64	ldq_u $1,8($16) # L : Latency: 3
61	sll $19,8,$7 # U : U L U L : 0x00000000 00aabb00	65	sll $19,8,$7 # U : U L U L : 0x00000000 00aabb00
62		66
		67	and $16,7,$6 # E : src misalignment
		68	ldq_u $5,15($16) # L : Latency: 3
63	zapnot $20,15,$20 # U : zero extend incoming csum	69	zapnot $20,15,$20 # U : zero extend incoming csum
64	ldq $2,0($17) # L : Latency: 3	70	ldq_u $2,0($17) # L : U L U L : Latency: 3
65	sll $19,24,$19 # U : U L L U : 0x000000aa bb000000	71
		72	extql $0,$6,$0 # U :
		73	extqh $1,$6,$22 # U :
		74	ldq_u $3,8($17) # L : Latency: 3
		75	sll $19,24,$19 # U : U U L U : 0x000000aa bb000000
		76
		77	cmoveq $6,$31,$22 # E : src aligned?
		78	ldq_u $23,15($17) # L : Latency: 3
66	inswl $18,3,$18 # U : 000000CCDD000000	79	inswl $18,3,$18 # U : 000000CCDD000000
		80	addl $19,$7,$19 # E : U L U L : <sign bits>bbaabb00
67		81
68	ldq $3,8($17) # L : Latency: 3	82	or $0,$22,$0 # E : 1st src word complete
69	bis $18,$4,$18 # E : 000000CCDDAABBCC	83	extql $1,$6,$1 # U :
70	addl $19,$7,$19 # E : <sign bits>bbaabb00	84	or $18,$4,$18 # E : 000000CCDDAABBCC
71	nop # E : U L U L	85	extqh $5,$6,$5 # U : L U L U
72		86
		87	and $17,7,$6 # E : dst misalignment
		88	extql $2,$6,$2 # U :
		89	or $1,$5,$1 # E : 2nd src word complete
		90	extqh $3,$6,$22 # U : L U L U :
		91
		92	cmoveq $6,$31,$22 # E : dst aligned?
		93	extql $3,$6,$3 # U :
73	addq $20,$0,$20 # E : begin summing the words	94	addq $20,$0,$20 # E : begin summing the words
		95	extqh $23,$6,$23 # U : L U L U :
		96
74	srl $18,16,$4 # U : 0000000000CCDDAA	97	srl $18,16,$4 # U : 0000000000CCDDAA
		98	or $2,$22,$2 # E : 1st dst word complete
75	zap $19,0x3,$19 # U : <sign bits>bbaa0000	99	zap $19,0x3,$19 # U : <sign bits>bbaa0000
76	nop # E : L U U L	100	or $3,$23,$3 # E : U L U L : 2nd dst word complete
77		101
78	cmpult $20,$0,$0 # E :	102	cmpult $20,$0,$0 # E :
79	addq $20,$1,$20 # E :	103	addq $20,$1,$20 # E :