aboutsummaryrefslogtreecommitdiffstats
path: root/arch/x86/crypto/aesni-intel_asm.S
diff options
context:
space:
mode:
Diffstat (limited to 'arch/x86/crypto/aesni-intel_asm.S')
-rw-r--r--arch/x86/crypto/aesni-intel_asm.S75
1 files changed, 48 insertions, 27 deletions
diff --git a/arch/x86/crypto/aesni-intel_asm.S b/arch/x86/crypto/aesni-intel_asm.S
index 6bd2c6c95373..383a6f84a060 100644
--- a/arch/x86/crypto/aesni-intel_asm.S
+++ b/arch/x86/crypto/aesni-intel_asm.S
@@ -31,6 +31,7 @@
31 31
32#include <linux/linkage.h> 32#include <linux/linkage.h>
33#include <asm/inst.h> 33#include <asm/inst.h>
34#include <asm/frame.h>
34 35
35/* 36/*
36 * The following macros are used to move an (un)aligned 16 byte value to/from 37 * The following macros are used to move an (un)aligned 16 byte value to/from
@@ -1800,11 +1801,12 @@ ENDPROC(_key_expansion_256b)
1800 * unsigned int key_len) 1801 * unsigned int key_len)
1801 */ 1802 */
1802ENTRY(aesni_set_key) 1803ENTRY(aesni_set_key)
1804 FRAME_BEGIN
1803#ifndef __x86_64__ 1805#ifndef __x86_64__
1804 pushl KEYP 1806 pushl KEYP
1805 movl 8(%esp), KEYP # ctx 1807 movl (FRAME_OFFSET+8)(%esp), KEYP # ctx
1806 movl 12(%esp), UKEYP # in_key 1808 movl (FRAME_OFFSET+12)(%esp), UKEYP # in_key
1807 movl 16(%esp), %edx # key_len 1809 movl (FRAME_OFFSET+16)(%esp), %edx # key_len
1808#endif 1810#endif
1809 movups (UKEYP), %xmm0 # user key (first 16 bytes) 1811 movups (UKEYP), %xmm0 # user key (first 16 bytes)
1810 movaps %xmm0, (KEYP) 1812 movaps %xmm0, (KEYP)
@@ -1905,6 +1907,7 @@ ENTRY(aesni_set_key)
1905#ifndef __x86_64__ 1907#ifndef __x86_64__
1906 popl KEYP 1908 popl KEYP
1907#endif 1909#endif
1910 FRAME_END
1908 ret 1911 ret
1909ENDPROC(aesni_set_key) 1912ENDPROC(aesni_set_key)
1910 1913
@@ -1912,12 +1915,13 @@ ENDPROC(aesni_set_key)
1912 * void aesni_enc(struct crypto_aes_ctx *ctx, u8 *dst, const u8 *src) 1915 * void aesni_enc(struct crypto_aes_ctx *ctx, u8 *dst, const u8 *src)
1913 */ 1916 */
1914ENTRY(aesni_enc) 1917ENTRY(aesni_enc)
1918 FRAME_BEGIN
1915#ifndef __x86_64__ 1919#ifndef __x86_64__
1916 pushl KEYP 1920 pushl KEYP
1917 pushl KLEN 1921 pushl KLEN
1918 movl 12(%esp), KEYP 1922 movl (FRAME_OFFSET+12)(%esp), KEYP # ctx
1919 movl 16(%esp), OUTP 1923 movl (FRAME_OFFSET+16)(%esp), OUTP # dst
1920 movl 20(%esp), INP 1924 movl (FRAME_OFFSET+20)(%esp), INP # src
1921#endif 1925#endif
1922 movl 480(KEYP), KLEN # key length 1926 movl 480(KEYP), KLEN # key length
1923 movups (INP), STATE # input 1927 movups (INP), STATE # input
@@ -1927,6 +1931,7 @@ ENTRY(aesni_enc)
1927 popl KLEN 1931 popl KLEN
1928 popl KEYP 1932 popl KEYP
1929#endif 1933#endif
1934 FRAME_END
1930 ret 1935 ret
1931ENDPROC(aesni_enc) 1936ENDPROC(aesni_enc)
1932 1937
@@ -2101,12 +2106,13 @@ ENDPROC(_aesni_enc4)
2101 * void aesni_dec (struct crypto_aes_ctx *ctx, u8 *dst, const u8 *src) 2106 * void aesni_dec (struct crypto_aes_ctx *ctx, u8 *dst, const u8 *src)
2102 */ 2107 */
2103ENTRY(aesni_dec) 2108ENTRY(aesni_dec)
2109 FRAME_BEGIN
2104#ifndef __x86_64__ 2110#ifndef __x86_64__
2105 pushl KEYP 2111 pushl KEYP
2106 pushl KLEN 2112 pushl KLEN
2107 movl 12(%esp), KEYP 2113 movl (FRAME_OFFSET+12)(%esp), KEYP # ctx
2108 movl 16(%esp), OUTP 2114 movl (FRAME_OFFSET+16)(%esp), OUTP # dst
2109 movl 20(%esp), INP 2115 movl (FRAME_OFFSET+20)(%esp), INP # src
2110#endif 2116#endif
2111 mov 480(KEYP), KLEN # key length 2117 mov 480(KEYP), KLEN # key length
2112 add $240, KEYP 2118 add $240, KEYP
@@ -2117,6 +2123,7 @@ ENTRY(aesni_dec)
2117 popl KLEN 2123 popl KLEN
2118 popl KEYP 2124 popl KEYP
2119#endif 2125#endif
2126 FRAME_END
2120 ret 2127 ret
2121ENDPROC(aesni_dec) 2128ENDPROC(aesni_dec)
2122 2129
@@ -2292,14 +2299,15 @@ ENDPROC(_aesni_dec4)
2292 * size_t len) 2299 * size_t len)
2293 */ 2300 */
2294ENTRY(aesni_ecb_enc) 2301ENTRY(aesni_ecb_enc)
2302 FRAME_BEGIN
2295#ifndef __x86_64__ 2303#ifndef __x86_64__
2296 pushl LEN 2304 pushl LEN
2297 pushl KEYP 2305 pushl KEYP
2298 pushl KLEN 2306 pushl KLEN
2299 movl 16(%esp), KEYP 2307 movl (FRAME_OFFSET+16)(%esp), KEYP # ctx
2300 movl 20(%esp), OUTP 2308 movl (FRAME_OFFSET+20)(%esp), OUTP # dst
2301 movl 24(%esp), INP 2309 movl (FRAME_OFFSET+24)(%esp), INP # src
2302 movl 28(%esp), LEN 2310 movl (FRAME_OFFSET+28)(%esp), LEN # len
2303#endif 2311#endif
2304 test LEN, LEN # check length 2312 test LEN, LEN # check length
2305 jz .Lecb_enc_ret 2313 jz .Lecb_enc_ret
@@ -2342,6 +2350,7 @@ ENTRY(aesni_ecb_enc)
2342 popl KEYP 2350 popl KEYP
2343 popl LEN 2351 popl LEN
2344#endif 2352#endif
2353 FRAME_END
2345 ret 2354 ret
2346ENDPROC(aesni_ecb_enc) 2355ENDPROC(aesni_ecb_enc)
2347 2356
@@ -2350,14 +2359,15 @@ ENDPROC(aesni_ecb_enc)
2350 * size_t len); 2359 * size_t len);
2351 */ 2360 */
2352ENTRY(aesni_ecb_dec) 2361ENTRY(aesni_ecb_dec)
2362 FRAME_BEGIN
2353#ifndef __x86_64__ 2363#ifndef __x86_64__
2354 pushl LEN 2364 pushl LEN
2355 pushl KEYP 2365 pushl KEYP
2356 pushl KLEN 2366 pushl KLEN
2357 movl 16(%esp), KEYP 2367 movl (FRAME_OFFSET+16)(%esp), KEYP # ctx
2358 movl 20(%esp), OUTP 2368 movl (FRAME_OFFSET+20)(%esp), OUTP # dst
2359 movl 24(%esp), INP 2369 movl (FRAME_OFFSET+24)(%esp), INP # src
2360 movl 28(%esp), LEN 2370 movl (FRAME_OFFSET+28)(%esp), LEN # len
2361#endif 2371#endif
2362 test LEN, LEN 2372 test LEN, LEN
2363 jz .Lecb_dec_ret 2373 jz .Lecb_dec_ret
@@ -2401,6 +2411,7 @@ ENTRY(aesni_ecb_dec)
2401 popl KEYP 2411 popl KEYP
2402 popl LEN 2412 popl LEN
2403#endif 2413#endif
2414 FRAME_END
2404 ret 2415 ret
2405ENDPROC(aesni_ecb_dec) 2416ENDPROC(aesni_ecb_dec)
2406 2417
@@ -2409,16 +2420,17 @@ ENDPROC(aesni_ecb_dec)
2409 * size_t len, u8 *iv) 2420 * size_t len, u8 *iv)
2410 */ 2421 */
2411ENTRY(aesni_cbc_enc) 2422ENTRY(aesni_cbc_enc)
2423 FRAME_BEGIN
2412#ifndef __x86_64__ 2424#ifndef __x86_64__
2413 pushl IVP 2425 pushl IVP
2414 pushl LEN 2426 pushl LEN
2415 pushl KEYP 2427 pushl KEYP
2416 pushl KLEN 2428 pushl KLEN
2417 movl 20(%esp), KEYP 2429 movl (FRAME_OFFSET+20)(%esp), KEYP # ctx
2418 movl 24(%esp), OUTP 2430 movl (FRAME_OFFSET+24)(%esp), OUTP # dst
2419 movl 28(%esp), INP 2431 movl (FRAME_OFFSET+28)(%esp), INP # src
2420 movl 32(%esp), LEN 2432 movl (FRAME_OFFSET+32)(%esp), LEN # len
2421 movl 36(%esp), IVP 2433 movl (FRAME_OFFSET+36)(%esp), IVP # iv
2422#endif 2434#endif
2423 cmp $16, LEN 2435 cmp $16, LEN
2424 jb .Lcbc_enc_ret 2436 jb .Lcbc_enc_ret
@@ -2443,6 +2455,7 @@ ENTRY(aesni_cbc_enc)
2443 popl LEN 2455 popl LEN
2444 popl IVP 2456 popl IVP
2445#endif 2457#endif
2458 FRAME_END
2446 ret 2459 ret
2447ENDPROC(aesni_cbc_enc) 2460ENDPROC(aesni_cbc_enc)
2448 2461
@@ -2451,16 +2464,17 @@ ENDPROC(aesni_cbc_enc)
2451 * size_t len, u8 *iv) 2464 * size_t len, u8 *iv)
2452 */ 2465 */
2453ENTRY(aesni_cbc_dec) 2466ENTRY(aesni_cbc_dec)
2467 FRAME_BEGIN
2454#ifndef __x86_64__ 2468#ifndef __x86_64__
2455 pushl IVP 2469 pushl IVP
2456 pushl LEN 2470 pushl LEN
2457 pushl KEYP 2471 pushl KEYP
2458 pushl KLEN 2472 pushl KLEN
2459 movl 20(%esp), KEYP 2473 movl (FRAME_OFFSET+20)(%esp), KEYP # ctx
2460 movl 24(%esp), OUTP 2474 movl (FRAME_OFFSET+24)(%esp), OUTP # dst
2461 movl 28(%esp), INP 2475 movl (FRAME_OFFSET+28)(%esp), INP # src
2462 movl 32(%esp), LEN 2476 movl (FRAME_OFFSET+32)(%esp), LEN # len
2463 movl 36(%esp), IVP 2477 movl (FRAME_OFFSET+36)(%esp), IVP # iv
2464#endif 2478#endif
2465 cmp $16, LEN 2479 cmp $16, LEN
2466 jb .Lcbc_dec_just_ret 2480 jb .Lcbc_dec_just_ret
@@ -2534,13 +2548,16 @@ ENTRY(aesni_cbc_dec)
2534 popl LEN 2548 popl LEN
2535 popl IVP 2549 popl IVP
2536#endif 2550#endif
2551 FRAME_END
2537 ret 2552 ret
2538ENDPROC(aesni_cbc_dec) 2553ENDPROC(aesni_cbc_dec)
2539 2554
2540#ifdef __x86_64__ 2555#ifdef __x86_64__
2556.pushsection .rodata
2541.align 16 2557.align 16
2542.Lbswap_mask: 2558.Lbswap_mask:
2543 .byte 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0 2559 .byte 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0
2560.popsection
2544 2561
2545/* 2562/*
2546 * _aesni_inc_init: internal ABI 2563 * _aesni_inc_init: internal ABI
@@ -2598,6 +2615,7 @@ ENDPROC(_aesni_inc)
2598 * size_t len, u8 *iv) 2615 * size_t len, u8 *iv)
2599 */ 2616 */
2600ENTRY(aesni_ctr_enc) 2617ENTRY(aesni_ctr_enc)
2618 FRAME_BEGIN
2601 cmp $16, LEN 2619 cmp $16, LEN
2602 jb .Lctr_enc_just_ret 2620 jb .Lctr_enc_just_ret
2603 mov 480(KEYP), KLEN 2621 mov 480(KEYP), KLEN
@@ -2651,6 +2669,7 @@ ENTRY(aesni_ctr_enc)
2651.Lctr_enc_ret: 2669.Lctr_enc_ret:
2652 movups IV, (IVP) 2670 movups IV, (IVP)
2653.Lctr_enc_just_ret: 2671.Lctr_enc_just_ret:
2672 FRAME_END
2654 ret 2673 ret
2655ENDPROC(aesni_ctr_enc) 2674ENDPROC(aesni_ctr_enc)
2656 2675
@@ -2677,6 +2696,7 @@ ENDPROC(aesni_ctr_enc)
2677 * bool enc, u8 *iv) 2696 * bool enc, u8 *iv)
2678 */ 2697 */
2679ENTRY(aesni_xts_crypt8) 2698ENTRY(aesni_xts_crypt8)
2699 FRAME_BEGIN
2680 cmpb $0, %cl 2700 cmpb $0, %cl
2681 movl $0, %ecx 2701 movl $0, %ecx
2682 movl $240, %r10d 2702 movl $240, %r10d
@@ -2777,6 +2797,7 @@ ENTRY(aesni_xts_crypt8)
2777 pxor INC, STATE4 2797 pxor INC, STATE4
2778 movdqu STATE4, 0x70(OUTP) 2798 movdqu STATE4, 0x70(OUTP)
2779 2799
2800 FRAME_END
2780 ret 2801 ret
2781ENDPROC(aesni_xts_crypt8) 2802ENDPROC(aesni_xts_crypt8)
2782 2803
generated by cgit v1.2.2 (git 2.25.0) at 2025-02-23 01:50:53 -0500