/* * INET An implementation of the TCP/IP protocol suite for the LINUX * operating system. INET is implemented using the BSD Socket * interface as the means of communication with the user level. * * IP/TCP/UDP checksumming routines * * Xtensa version: Copyright (C) 2001 Tensilica, Inc. by Kevin Chea * Optimized by Joe Taylor * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version * 2 of the License, or (at your option) any later version. */ #include <asm/errno.h> #include <linux/linkage.h> #include <variant/core.h> /* * computes a partial checksum, e.g. for TCP/UDP fragments */ /* * unsigned int csum_partial(const unsigned char *buf, int len, * unsigned int sum); * a2 = buf * a3 = len * a4 = sum * * This function assumes 2- or 4-byte alignment. Other alignments will fail! */ /* ONES_ADD converts twos-complement math to ones-complement. */ #define ONES_ADD(sum, val) \ add sum, sum, val ; \ bgeu sum, val, 99f ; \ addi sum, sum, 1 ; \ 99: ; .text ENTRY(csum_partial) /* * Experiments with Ethernet and SLIP connections show that buf * is aligned on either a 2-byte or 4-byte boundary. */ entry sp, 32 extui a5, a2, 0, 2 bnez a5, 8f /* branch if 2-byte aligned */ /* Fall-through on common case, 4-byte alignment */ 1: srli a5, a3, 5 /* 32-byte chunks */ #if XCHAL_HAVE_LOOPS loopgtz a5, 2f #else beqz a5, 2f slli a5, a5, 5 add a5, a5, a2 /* a5 = end of last 32-byte chunk */ .Loop1: #endif l32i a6, a2, 0 l32i a7, a2, 4 ONES_ADD(a4, a6) ONES_ADD(a4, a7) l32i a6, a2, 8 l32i a7, a2, 12 ONES_ADD(a4, a6) ONES_ADD(a4, a7) l32i a6, a2, 16 l32i a7, a2, 20 ONES_ADD(a4, a6) ONES_ADD(a4, a7) l32i a6, a2, 24 l32i a7, a2, 28 ONES_ADD(a4, a6) ONES_ADD(a4, a7) addi a2, a2, 4*8 #if !XCHAL_HAVE_LOOPS blt a2, a5, .Loop1 #endif 2: extui a5, a3, 2, 3 /* remaining 4-byte chunks */ #if XCHAL_HAVE_LOOPS loopgtz a5, 3f #else beqz a5, 3f slli a5, a5, 2 add a5, a5, a2 /* a5 = end of last 4-byte chunk */ .Loop2: #endif l32i a6, a2, 0 ONES_ADD(a4, a6) addi a2, a2, 4 #if !XCHAL_HAVE_LOOPS blt a2, a5, .Loop2 #endif 3: _bbci.l a3, 1, 5f /* remaining 2-byte chunk */ l16ui a6, a2, 0 ONES_ADD(a4, a6) addi a2, a2, 2 5: _bbci.l a3, 0, 7f /* remaining 1-byte chunk */ 6: l8ui a6, a2, 0 #ifdef __XTENSA_EB__ slli a6, a6, 8 /* load byte into bits 8..15 */ #endif ONES_ADD(a4, a6) 7: mov a2, a4 retw /* uncommon case, buf is 2-byte aligned */ 8: beqz a3, 7b /* branch if len == 0 */ beqi a3, 1, 6b /* branch if len == 1 */ extui a5, a2, 0, 1 bnez a5, 8f /* branch if 1-byte aligned */ l16ui a6, a2, 0 /* common case, len >= 2 */ ONES_ADD(a4, a6) addi a2, a2, 2 /* adjust buf */ addi a3, a3, -2 /* adjust len */ j 1b /* now buf is 4-byte aligned */ /* case: odd-byte aligned, len > 1 * This case is dog slow, so don't give us an odd address. * (I don't think this ever happens, but just in case.) */ 8: srli a5, a3, 2 /* 4-byte chunks */ #if XCHAL_HAVE_LOOPS loopgtz a5, 2f #else beqz a5, 2f slli a5, a5, 2 add a5, a5, a2 /* a5 = end of last 4-byte chunk */ .Loop3: #endif l8ui a6, a2, 0 /* bits 24..31 */ l16ui a7, a2, 1 /* bits 8..23 */ l8ui a8, a2, 3 /* bits 0.. 8 */ #ifdef __XTENSA_EB__ slli a6, a6, 24 #else slli a8, a8, 24 #endif slli a7, a7, 8 or a7, a7, a6 or a7, a7, a8 ONES_ADD(a4, a7) addi a2, a2, 4 #if !XCHAL_HAVE_LOOPS blt a2, a5, .Loop3 #endif 2: _bbci.l a3, 1, 3f /* remaining 2-byte chunk, still odd addr */ l8ui a6, a2, 0 l8ui a7, a2, 1 #ifdef __XTENSA_EB__ slli a6, a6, 8 #else slli a7, a7, 8 #endif or a7, a7, a6 ONES_ADD(a4, a7) addi a2, a2, 2 3: j 5b /* branch to handle the remaining byte */ /* * Copy from ds while checksumming, otherwise like csum_partial * * The macros SRC and DST specify the type of access for the instruction. * thus we can call a custom exception handler for each access type. */ #define SRC(y...) \ 9999: y; \ .section __ex_table, "a"; \ .long 9999b, 6001f ; \ .previous #define DST(y...) \ 9999: y; \ .section __ex_table, "a"; \ .long 9999b, 6002f ; \ .previous /* unsigned int csum_partial_copy_generic (const char *src, char *dst, int len, int sum, int *src_err_ptr, int *dst_err_ptr) a2 = src a3 = dst a4 = len a5 = sum a6 = src_err_ptr a7 = dst_err_ptr a8 = temp a9 = temp a10 = temp a11 = original len for exception handling a12 = original dst for exception handling This function is optimized for 4-byte aligned addresses. Other alignments work, but not nearly as efficiently. */ ENTRY(csum_partial_copy_generic) entry sp, 32 mov a12, a3 mov a11, a4 or a10, a2, a3 /* We optimize the following alignment tests for the 4-byte aligned case. Two bbsi.l instructions might seem more optimal (commented out below). However, both labels 5: and 3: are out of the imm8 range, so the assembler relaxes them into equivalent bbci.l, j combinations, which is actually slower. */ extui a9, a10, 0, 2 beqz a9, 1f /* branch if both are 4-byte aligned */ bbsi.l a10, 0, 5f /* branch if one address is odd */ j 3f /* one address is 2-byte aligned */ /* _bbsi.l a10, 0, 5f */ /* branch if odd address */ /* _bbsi.l a10, 1, 3f */ /* branch if 2-byte-aligned address */ 1: /* src and dst are both 4-byte aligned */ srli a10, a4, 5 /* 32-byte chunks */ #if XCHAL_HAVE_LOOPS loopgtz a10, 2f #else beqz a10, 2f slli a10, a10, 5 add a10, a10, a2 /* a10 = end of last 32-byte src chunk */ .Loop5: #endif SRC( l32i a9, a2, 0 ) SRC( l32i a8, a2, 4 ) DST( s32i a9, a3, 0 ) DST( s32i a8, a3, 4 ) ONES_ADD(a5, a9) ONES_ADD(a5, a8) SRC( l32i a9, a2, 8 ) SRC( l32i a8, a2, 12 ) DST( s32i a9, a3, 8 ) DST( s32i a8, a3, 12 ) ONES_ADD(a5, a9) ONES_ADD(a5, a8) SRC( l32i a9, a2, 16 ) SRC( l32i a8, a2, 20 ) DST( s32i a9, a3, 16 ) DST( s32i a8, a3, 20 ) ONES_ADD(a5, a9) ONES_ADD(a5, a8) SRC( l32i a9, a2, 24 ) SRC( l32i a8, a2, 28 ) DST( s32i a9, a3, 24 ) DST( s32i a8, a3, 28 ) ONES_ADD(a5, a9) ONES_ADD(a5, a8) addi a2, a2, 32 addi a3, a3, 32 #if !XCHAL_HAVE_LOOPS blt a2, a10, .Loop5 #endif 2: extui a10, a4, 2, 3 /* remaining 4-byte chunks */ extui a4, a4, 0, 2 /* reset len for general-case, 2-byte chunks */ #if XCHAL_HAVE_LOOPS loopgtz a10, 3f #else beqz a10, 3f slli a10, a10, 2 add a10, a10, a2 /* a10 = end of last 4-byte src chunk */ .Loop6: #endif SRC( l32i a9, a2, 0 ) DST( s32i a9, a3, 0 ) ONES_ADD(a5, a9) addi a2, a2, 4 addi a3, a3, 4 #if !XCHAL_HAVE_LOOPS blt a2, a10, .Loop6 #endif 3: /* Control comes to here in two cases: (1) It may fall through to here from the 4-byte alignment case to process, at most, one 2-byte chunk. (2) It branches to here from above if either src or dst is 2-byte aligned, and we process all bytes here, except for perhaps a trailing odd byte. It's inefficient, so align your addresses to 4-byte boundaries. a2 = src a3 = dst a4 = len a5 = sum */ srli a10, a4, 1 /* 2-byte chunks */ #if XCHAL_HAVE_LOOPS loopgtz a10, 4f #else beqz a10, 4f slli a10, a10, 1 add a10, a10, a2 /* a10 = end of last 2-byte src chunk */ .Loop7: #endif SRC( l16ui a9, a2, 0 ) DST( s16i a9, a3, 0 ) ONES_ADD(a5, a9) addi a2, a2, 2 addi a3, a3, 2 #if !XCHAL_HAVE_LOOPS blt a2, a10, .Loop7 #endif 4: /* This section processes a possible trailing odd byte. */ _bbci.l a4, 0, 8f /* 1-byte chunk */ SRC( l8ui a9, a2, 0 ) DST( s8i a9, a3, 0 ) #ifdef __XTENSA_EB__ slli a9, a9, 8 /* shift byte to bits 8..15 */ #endif ONES_ADD(a5, a9) 8: mov a2, a5 retw 5: /* Control branch to here when either src or dst is odd. We process all bytes using 8-bit accesses. Grossly inefficient, so don't feed us an odd address. */ srli a10, a4, 1 /* handle in pairs for 16-bit csum */ #if XCHAL_HAVE_LOOPS loopgtz a10, 6f #else beqz a10, 6f slli a10, a10, 1 add a10, a10, a2 /* a10 = end of last odd-aligned, 2-byte src chunk */ .Loop8: #endif SRC( l8ui a9, a2, 0 ) SRC( l8ui a8, a2, 1 ) DST( s8i a9, a3, 0 ) DST( s8i a8, a3, 1 ) #ifdef __XTENSA_EB__ slli a9, a9, 8 /* combine into a single 16-bit value */ #else /* for checksum computation */ slli a8, a8, 8 #endif or a9, a9, a8 ONES_ADD(a5, a9) addi a2, a2, 2 addi a3, a3, 2 #if !XCHAL_HAVE_LOOPS blt a2, a10, .Loop8 #endif 6: j 4b /* process the possible trailing odd byte */ # Exception handler: .section .fixup, "ax" /* a6 = src_err_ptr a7 = dst_err_ptr a11 = original len for exception handling a12 = original dst for exception handling */ 6001: _movi a2, -EFAULT s32i a2, a6, 0 /* src_err_ptr */ # clear the complete destination - computing the rest # is too much work movi a2, 0 #if XCHAL_HAVE_LOOPS loopgtz a11, 2f #else beqz a11, 2f add a11, a11, a12 /* a11 = ending address */ .Leloop: #endif s8i a2, a12, 0 addi a12, a12, 1 #if !XCHAL_HAVE_LOOPS blt a12, a11, .Leloop #endif 2: retw 6002: movi a2, -EFAULT s32i a2, a7, 0 /* dst_err_ptr */ movi a2, 0 retw .previous