/*
* INET An implementation of the TCP/IP protocol suite for the LINUX
* operating system. INET is implemented using the BSD Socket
* interface as the means of communication with the user level.
*
* IP/TCP/UDP checksumming routines
*
* Xtensa version: Copyright (C) 2001 Tensilica, Inc. by Kevin Chea
* Optimized by Joe Taylor
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version
* 2 of the License, or (at your option) any later version.
*/
#include <asm/errno.h>
#include <linux/linkage.h>
#include <variant/core.h>
/*
* computes a partial checksum, e.g. for TCP/UDP fragments
*/
/*
* unsigned int csum_partial(const unsigned char *buf, int len,
* unsigned int sum);
* a2 = buf
* a3 = len
* a4 = sum
*
* This function assumes 2- or 4-byte alignment. Other alignments will fail!
*/
/* ONES_ADD converts twos-complement math to ones-complement. */
#define ONES_ADD(sum, val) \
add sum, sum, val ; \
bgeu sum, val, 99f ; \
addi sum, sum, 1 ; \
99: ;
.text
ENTRY(csum_partial)
/*
* Experiments with Ethernet and SLIP connections show that buf
* is aligned on either a 2-byte or 4-byte boundary.
*/
entry sp, 32
extui a5, a2, 0, 2
bnez a5, 8f /* branch if 2-byte aligned */
/* Fall-through on common case, 4-byte alignment */
1:
srli a5, a3, 5 /* 32-byte chunks */
#if XCHAL_HAVE_LOOPS
loopgtz a5, 2f
#else
beqz a5, 2f
slli a5, a5, 5
add a5, a5, a2 /* a5 = end of last 32-byte chunk */
.Loop1:
#endif
l32i a6, a2, 0
l32i a7, a2, 4
ONES_ADD(a4, a6)
ONES_ADD(a4, a7)
l32i a6, a2, 8
l32i a7, a2, 12
ONES_ADD(a4, a6)
ONES_ADD(a4, a7)
l32i a6, a2, 16
l32i a7, a2, 20
ONES_ADD(a4, a6)
ONES_ADD(a4, a7)
l32i a6, a2, 24
l32i a7, a2, 28
ONES_ADD(a4, a6)
ONES_ADD(a4, a7)
addi a2, a2, 4*8
#if !XCHAL_HAVE_LOOPS
blt a2, a5, .Loop1
#endif
2:
extui a5, a3, 2, 3 /* remaining 4-byte chunks */
#if XCHAL_HAVE_LOOPS
loopgtz a5, 3f
#else
beqz a5, 3f
slli a5, a5, 2
add a5, a5, a2 /* a5 = end of last 4-byte chunk */
.Loop2:
#endif
l32i a6, a2, 0
ONES_ADD(a4, a6)
addi a2, a2, 4
#if !XCHAL_HAVE_LOOPS
blt a2, a5, .Loop2
#endif
3:
_bbci.l a3, 1, 5f /* remaining 2-byte chunk */
l16ui a6, a2, 0
ONES_ADD(a4, a6)
addi a2, a2, 2
5:
_bbci.l a3, 0, 7f /* remaining 1-byte chunk */
6: l8ui a6, a2, 0
#ifdef __XTENSA_EB__
slli a6, a6, 8 /* load byte into bits 8..15 */
#endif
ONES_ADD(a4, a6)
7:
mov a2, a4
retw
/* uncommon case, buf is 2-byte aligned */
8:
beqz a3, 7b /* branch if len == 0 */
beqi a3, 1, 6b /* branch if len == 1 */
extui a5, a2, 0, 1
bnez a5, 8f /* branch if 1-byte aligned */
l16ui a6, a2, 0 /* common case, len >= 2 */
ONES_ADD(a4, a6)
addi a2, a2, 2 /* adjust buf */
addi a3, a3, -2 /* adjust len */
j 1b /* now buf is 4-byte aligned */
/* case: odd-byte aligned, len > 1
* This case is dog slow, so don't give us an odd address.
* (I don't think this ever happens, but just in case.)
*/
8:
srli a5, a3, 2 /* 4-byte chunks */
#if XCHAL_HAVE_LOOPS
loopgtz a5, 2f
#else
beqz a5, 2f
slli a5, a5, 2
add a5, a5, a2 /* a5 = end of last 4-byte chunk */
.Loop3:
#endif
l8ui a6, a2, 0 /* bits 24..31 */
l16ui a7, a2, 1 /* bits 8..23 */
l8ui a8, a2, 3 /* bits 0.. 8 */
#ifdef __XTENSA_EB__
slli a6, a6, 24
#else
slli a8, a8, 24
#endif
slli a7, a7, 8
or a7, a7, a6
or a7, a7, a8
ONES_ADD(a4, a7)
addi a2, a2, 4
#if !XCHAL_HAVE_LOOPS
blt a2, a5, .Loop3
#endif
2:
_bbci.l a3, 1, 3f /* remaining 2-byte chunk, still odd addr */
l8ui a6, a2, 0
l8ui a7, a2, 1
#ifdef __XTENSA_EB__
slli a6, a6, 8
#else
slli a7, a7, 8
#endif
or a7, a7, a6
ONES_ADD(a4, a7)
addi a2, a2, 2
3:
j 5b /* branch to handle the remaining byte */
/*
* Copy from ds while checksumming, otherwise like csum_partial
*
* The macros SRC and DST specify the type of access for the instruction.
* thus we can call a custom exception handler for each access type.
*/
#define SRC(y...) \
9999: y; \
.section __ex_table, "a"; \
.long 9999b, 6001f ; \
.previous
#define DST(y...) \
9999: y; \
.section __ex_table, "a"; \
.long 9999b, 6002f ; \
.previous
/*
unsigned int csum_partial_copy_generic (const char *src, char *dst, int len,
int sum, int *src_err_ptr, int *dst_err_ptr)
a2 = src
a3 = dst
a4 = len
a5 = sum
a6 = src_err_ptr
a7 = dst_err_ptr
a8 = temp
a9 = temp
a10 = temp
a11 = original len for exception handling
a12 = original dst for exception handling
This function is optimized for 4-byte aligned addresses. Other
alignments work, but not nearly as efficiently.
*/
ENTRY(csum_partial_copy_generic)
entry sp, 32
mov a12, a3
mov a11, a4
or a10, a2, a3
/* We optimize the following alignment tests for the 4-byte
aligned case. Two bbsi.l instructions might seem more optimal
(commented out below). However, both labels 5: and 3: are out
of the imm8 range, so the assembler relaxes them into
equivalent bbci.l, j combinations, which is actually
slower. */
extui a9, a10, 0, 2
beqz a9, 1f /* branch if both are 4-byte aligned */
bbsi.l a10, 0, 5f /* branch if one address is odd */
j 3f /* one address is 2-byte aligned */
/* _bbsi.l a10, 0, 5f */ /* branch if odd address */
/* _bbsi.l a10, 1, 3f */ /* branch if 2-byte-aligned address */
1:
/* src and dst are both 4-byte aligned */
srli a10, a4, 5 /* 32-byte chunks */
#if XCHAL_HAVE_LOOPS
loopgtz a10, 2f
#else
beqz a10, 2f
slli a10, a10, 5
add a10, a10, a2 /* a10 = end of last 32-byte src chunk */
.Loop5:
#endif
SRC( l32i a9, a2, 0 )
SRC( l32i a8, a2, 4 )
DST( s32i a9, a3, 0 )
DST( s32i a8, a3, 4 )
ONES_ADD(a5, a9)
ONES_ADD(a5, a8)
SRC( l32i a9, a2, 8 )
SRC( l32i a8, a2, 12 )
DST( s32i a9, a3, 8 )
DST( s32i a8, a3, 12 )
ONES_ADD(a5, a9)
ONES_ADD(a5, a8)
SRC( l32i a9, a2, 16 )
SRC( l32i a8, a2, 20 )
DST( s32i a9, a3, 16 )
DST( s32i a8, a3, 20 )
ONES_ADD(a5, a9)
ONES_ADD(a5, a8)
SRC( l32i a9, a2, 24 )
SRC( l32i a8, a2, 28 )
DST( s32i a9, a3, 24 )
DST( s32i a8, a3, 28 )
ONES_ADD(a5, a9)
ONES_ADD(a5, a8)
addi a2, a2, 32
addi a3, a3, 32
#if !XCHAL_HAVE_LOOPS
blt a2, a10, .Loop5
#endif
2:
extui a10, a4, 2, 3 /* remaining 4-byte chunks */
extui a4, a4, 0, 2 /* reset len for general-case, 2-byte chunks */
#if XCHAL_HAVE_LOOPS
loopgtz a10, 3f
#else
beqz a10, 3f
slli a10, a10, 2
add a10, a10, a2 /* a10 = end of last 4-byte src chunk */
.Loop6:
#endif
SRC( l32i a9, a2, 0 )
DST( s32i a9, a3, 0 )
ONES_ADD(a5, a9)
addi a2, a2, 4
addi a3, a3, 4
#if !XCHAL_HAVE_LOOPS
blt a2, a10, .Loop6
#endif
3:
/*
Control comes to here in two cases: (1) It may fall through
to here from the 4-byte alignment case to process, at most,
one 2-byte chunk. (2) It branches to here from above if
either src or dst is 2-byte aligned, and we process all bytes
here, except for perhaps a trailing odd byte. It's
inefficient, so align your addresses to 4-byte boundaries.
a2 = src
a3 = dst
a4 = len
a5 = sum
*/
srli a10, a4, 1 /* 2-byte chunks */
#if XCHAL_HAVE_LOOPS
loopgtz a10, 4f
#else
beqz a10, 4f
slli a10, a10, 1
add a10, a10, a2 /* a10 = end of last 2-byte src chunk */
.Loop7:
#endif
SRC( l16ui a9, a2, 0 )
DST( s16i a9, a3, 0 )
ONES_ADD(a5, a9)
addi a2, a2, 2
addi a3, a3, 2
#if !XCHAL_HAVE_LOOPS
blt a2, a10, .Loop7
#endif
4:
/* This section processes a possible trailing odd byte. */
_bbci.l a4, 0, 8f /* 1-byte chunk */
SRC( l8ui a9, a2, 0 )
DST( s8i a9, a3, 0 )
#ifdef __XTENSA_EB__
slli a9, a9, 8 /* shift byte to bits 8..15 */
#endif
ONES_ADD(a5, a9)
8:
mov a2, a5
retw
5:
/* Control branch to here when either src or dst is odd. We
process all bytes using 8-bit accesses. Grossly inefficient,
so don't feed us an odd address. */
srli a10, a4, 1 /* handle in pairs for 16-bit csum */
#if XCHAL_HAVE_LOOPS
loopgtz a10, 6f
#else
beqz a10, 6f
slli a10, a10, 1
add a10, a10, a2 /* a10 = end of last odd-aligned, 2-byte src chunk */
.Loop8:
#endif
SRC( l8ui a9, a2, 0 )
SRC( l8ui a8, a2, 1 )
DST( s8i a9, a3, 0 )
DST( s8i a8, a3, 1 )
#ifdef __XTENSA_EB__
slli a9, a9, 8 /* combine into a single 16-bit value */
#else /* for checksum computation */
slli a8, a8, 8
#endif
or a9, a9, a8
ONES_ADD(a5, a9)
addi a2, a2, 2
addi a3, a3, 2
#if !XCHAL_HAVE_LOOPS
blt a2, a10, .Loop8
#endif
6:
j 4b /* process the possible trailing odd byte */
# Exception handler:
.section .fixup, "ax"
/*
a6 = src_err_ptr
a7 = dst_err_ptr
a11 = original len for exception handling
a12 = original dst for exception handling
*/
6001:
_movi a2, -EFAULT
s32i a2, a6, 0 /* src_err_ptr */
# clear the complete destination - computing the rest
# is too much work
movi a2, 0
#if XCHAL_HAVE_LOOPS
loopgtz a11, 2f
#else
beqz a11, 2f
add a11, a11, a12 /* a11 = ending address */
.Leloop:
#endif
s8i a2, a12, 0
addi a12, a12, 1
#if !XCHAL_HAVE_LOOPS
blt a12, a11, .Leloop
#endif
2:
retw
6002:
movi a2, -EFAULT
s32i a2, a7, 0 /* dst_err_ptr */
movi a2, 0
retw
.previous
