1 files changed, 302 insertions, 0 deletions
diff --git a/arch/x86/crypto/sha1_ni_asm.S b/arch/x86/crypto/sha1_ni_asm.S
new file mode 100644
index 000000000000..874a651b9e7d
--- /dev/null
+++ b/arch/x86/crypto/sha1_ni_asm.S
@@ -0,0 +1,302 @@
+/*
+ * Intel SHA Extensions optimized implementation of a SHA-1 update function
+ *
+ * This file is provided under a dual BSD/GPLv2 license.  When using or
+ * redistributing this file, you may do so under either license.
+ *
+ * GPL LICENSE SUMMARY
+ *
+ * Copyright(c) 2015 Intel Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * Contact Information:
+ *      Sean Gulley <sean.m.gulley@intel.com>
+ *      Tim Chen <tim.c.chen@linux.intel.com>
+ *
+ * BSD LICENSE
+ *
+ * Copyright(c) 2015 Intel Corporation.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ *      * Redistributions of source code must retain the above copyright
+ *        notice, this list of conditions and the following disclaimer.
+ *      * Redistributions in binary form must reproduce the above copyright
+ *        notice, this list of conditions and the following disclaimer in
+ *        the documentation and/or other materials provided with the
+ *        distribution.
+ *      * Neither the name of Intel Corporation nor the names of its
+ *        contributors may be used to endorse or promote products derived
+ *        from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+#include <linux/linkage.h>
+#define DIGEST_PTR      %rdi    /* 1st arg */
+#define DATA_PTR        %rsi    /* 2nd arg */
+#define NUM_BLKS        %rdx    /* 3rd arg */
+#define RSPSAVE         %rax
+/* gcc conversion */
+#define FRAME_SIZE      32      /* space for 2x16 bytes */
+#define ABCD            %xmm0
+#define E0              %xmm1   /* Need two E's b/c they ping pong */
+#define E1              %xmm2
+#define MSG0            %xmm3
+#define MSG1            %xmm4
+#define MSG2            %xmm5
+#define MSG3            %xmm6
+#define SHUF_MASK       %xmm7
+/*
+ * Intel SHA Extensions optimized implementation of a SHA-1 update function
+ *
+ * The function takes a pointer to the current hash values, a pointer to the
+ * input data, and a number of 64 byte blocks to process.  Once all blocks have
+ * been processed, the digest pointer is  updated with the resulting hash value.
+ * The function only processes complete blocks, there is no functionality to
+ * store partial blocks. All message padding and hash value initialization must
+ * be done outside the update function.
+ *
+ * The indented lines in the loop are instructions related to rounds processing.
+ * The non-indented lines are instructions related to the message schedule.
+ *
+ * void sha1_ni_transform(uint32_t *digest, const void *data,
+                uint32_t numBlocks)
+ * digest : pointer to digest
+ * data: pointer to input data
+ * numBlocks: Number of blocks to process
+ */
+.text
+.align 32
+ENTRY(sha1_ni_transform)
+        mov             %rsp, RSPSAVE
+        sub             $FRAME_SIZE, %rsp
+        and             $~0xF, %rsp
+        shl             $6, NUM_BLKS            /* convert to bytes */
+        jz              .Ldone_hash
+        add             DATA_PTR, NUM_BLKS      /* pointer to end of data */
+        /* load initial hash values */
+        pinsrd          $3, 1*16(DIGEST_PTR), E0
+        movdqu          0*16(DIGEST_PTR), ABCD
+        pand            UPPER_WORD_MASK(%rip), E0
+        pshufd          $0x1B, ABCD, ABCD
+        movdqa          PSHUFFLE_BYTE_FLIP_MASK(%rip), SHUF_MASK
+.Lloop0:
+        /* Save hash values for addition after rounds */
+        movdqa          E0, (0*16)(%rsp)
+        movdqa          ABCD, (1*16)(%rsp)
+        /* Rounds 0-3 */
+        movdqu          0*16(DATA_PTR), MSG0
+        pshufb          SHUF_MASK, MSG0
+                paddd           MSG0, E0
+                movdqa          ABCD, E1
+                sha1rnds4       $0, E0, ABCD
+        /* Rounds 4-7 */
+        movdqu          1*16(DATA_PTR), MSG1
+        pshufb          SHUF_MASK, MSG1
+                sha1nexte       MSG1, E1
+                movdqa          ABCD, E0
+                sha1rnds4       $0, E1, ABCD
+        sha1msg1        MSG1, MSG0
+        /* Rounds 8-11 */
+        movdqu          2*16(DATA_PTR), MSG2
+        pshufb          SHUF_MASK, MSG2
+                sha1nexte       MSG2, E0
+                movdqa          ABCD, E1
+                sha1rnds4       $0, E0, ABCD
+        sha1msg1        MSG2, MSG1
+        pxor            MSG2, MSG0
+        /* Rounds 12-15 */
+        movdqu          3*16(DATA_PTR), MSG3
+        pshufb          SHUF_MASK, MSG3
+                sha1nexte       MSG3, E1
+                movdqa          ABCD, E0
+        sha1msg2        MSG3, MSG0
+                sha1rnds4       $0, E1, ABCD
+        sha1msg1        MSG3, MSG2
+        pxor            MSG3, MSG1
+        /* Rounds 16-19 */
+                sha1nexte       MSG0, E0
+                movdqa          ABCD, E1
+        sha1msg2        MSG0, MSG1
+                sha1rnds4       $0, E0, ABCD
+        sha1msg1        MSG0, MSG3
+        pxor            MSG0, MSG2
+        /* Rounds 20-23 */
+                sha1nexte       MSG1, E1
+                movdqa          ABCD, E0
+        sha1msg2        MSG1, MSG2
+                sha1rnds4       $1, E1, ABCD
+        sha1msg1        MSG1, MSG0
+        pxor            MSG1, MSG3
+        /* Rounds 24-27 */
+                sha1nexte       MSG2, E0
+                movdqa          ABCD, E1
+        sha1msg2        MSG2, MSG3
+                sha1rnds4       $1, E0, ABCD
+        sha1msg1        MSG2, MSG1
+        pxor            MSG2, MSG0
+        /* Rounds 28-31 */
+                sha1nexte       MSG3, E1
+                movdqa          ABCD, E0
+        sha1msg2        MSG3, MSG0
+                sha1rnds4       $1, E1, ABCD
+        sha1msg1        MSG3, MSG2
+        pxor            MSG3, MSG1
+        /* Rounds 32-35 */
+                sha1nexte       MSG0, E0
+                movdqa          ABCD, E1
+        sha1msg2        MSG0, MSG1
+                sha1rnds4       $1, E0, ABCD
+        sha1msg1        MSG0, MSG3
+        pxor            MSG0, MSG2
+        /* Rounds 36-39 */
+                sha1nexte       MSG1, E1
+                movdqa          ABCD, E0
+        sha1msg2        MSG1, MSG2
+                sha1rnds4       $1, E1, ABCD
+        sha1msg1        MSG1, MSG0
+        pxor            MSG1, MSG3
+        /* Rounds 40-43 */
+                sha1nexte       MSG2, E0
+                movdqa          ABCD, E1
+        sha1msg2        MSG2, MSG3
+                sha1rnds4       $2, E0, ABCD
+        sha1msg1        MSG2, MSG1
+        pxor            MSG2, MSG0
+        /* Rounds 44-47 */
+                sha1nexte       MSG3, E1
+                movdqa          ABCD, E0
+        sha1msg2        MSG3, MSG0
+                sha1rnds4       $2, E1, ABCD
+        sha1msg1        MSG3, MSG2
+        pxor            MSG3, MSG1
+        /* Rounds 48-51 */
+                sha1nexte       MSG0, E0
+                movdqa          ABCD, E1
+        sha1msg2        MSG0, MSG1
+                sha1rnds4       $2, E0, ABCD
+        sha1msg1        MSG0, MSG3
+        pxor            MSG0, MSG2
+        /* Rounds 52-55 */
+                sha1nexte       MSG1, E1
+                movdqa          ABCD, E0
+        sha1msg2        MSG1, MSG2
+                sha1rnds4       $2, E1, ABCD
+        sha1msg1        MSG1, MSG0
+        pxor            MSG1, MSG3
+        /* Rounds 56-59 */
+                sha1nexte       MSG2, E0
+                movdqa          ABCD, E1
+        sha1msg2        MSG2, MSG3
+                sha1rnds4       $2, E0, ABCD
+        sha1msg1        MSG2, MSG1
+        pxor            MSG2, MSG0
+        /* Rounds 60-63 */
+                sha1nexte       MSG3, E1
+                movdqa          ABCD, E0
+        sha1msg2        MSG3, MSG0
+                sha1rnds4       $3, E1, ABCD
+        sha1msg1        MSG3, MSG2
+        pxor            MSG3, MSG1
+        /* Rounds 64-67 */
+                sha1nexte       MSG0, E0
+                movdqa          ABCD, E1
+        sha1msg2        MSG0, MSG1
+                sha1rnds4       $3, E0, ABCD
+        sha1msg1        MSG0, MSG3
+        pxor            MSG0, MSG2
+        /* Rounds 68-71 */
+                sha1nexte       MSG1, E1
+                movdqa          ABCD, E0
+        sha1msg2        MSG1, MSG2
+                sha1rnds4       $3, E1, ABCD
+        pxor            MSG1, MSG3
+        /* Rounds 72-75 */
+                sha1nexte       MSG2, E0
+                movdqa          ABCD, E1
+        sha1msg2        MSG2, MSG3
+                sha1rnds4       $3, E0, ABCD
+        /* Rounds 76-79 */
+                sha1nexte       MSG3, E1
+                movdqa          ABCD, E0
+                sha1rnds4       $3, E1, ABCD
+        /* Add current hash values with previously saved */
+        sha1nexte       (0*16)(%rsp), E0
+        paddd           (1*16)(%rsp), ABCD
+        /* Increment data pointer and loop if more to process */
+        add             $64, DATA_PTR
+        cmp             NUM_BLKS, DATA_PTR
+        jne             .Lloop0
+        /* Write hash values back in the correct order */
+        pshufd          $0x1B, ABCD, ABCD
+        movdqu          ABCD, 0*16(DIGEST_PTR)
+        pextrd          $3, E0, 1*16(DIGEST_PTR)
+.Ldone_hash:
+        mov             RSPSAVE, %rsp
+        ret
+ENDPROC(sha1_ni_transform)
+.data
+.align 64
+PSHUFFLE_BYTE_FLIP_MASK:
+        .octa 0x000102030405060708090a0b0c0d0e0f
+UPPER_WORD_MASK:
+        .octa 0xFFFFFFFF000000000000000000000000

diff --git a/arch/x86/crypto/sha1_ni_asm.S b/arch/x86/crypto/sha1_ni_asm.S new file mode 100644 index 000000000000..874a651b9e7d --- /dev/null +++ b/arch/x86/crypto/sha1_ni_asm.S
@@ -0,0 +1,302 @@
	1	/*
	2	* Intel SHA Extensions optimized implementation of a SHA-1 update function
	3	*
	4	* This file is provided under a dual BSD/GPLv2 license. When using or
	5	* redistributing this file, you may do so under either license.
	6	*
	7	* GPL LICENSE SUMMARY
	8	*
	9	* Copyright(c) 2015 Intel Corporation.
	10	*
	11	* This program is free software; you can redistribute it and/or modify
	12	* it under the terms of version 2 of the GNU General Public License as
	13	* published by the Free Software Foundation.
	14	*
	15	* This program is distributed in the hope that it will be useful, but
	16	* WITHOUT ANY WARRANTY; without even the implied warranty of
	17	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	18	* General Public License for more details.
	19	*
	20	* Contact Information:
	21	* Sean Gulley <sean.m.gulley@intel.com>
	22	* Tim Chen <tim.c.chen@linux.intel.com>
	23	*
	24	* BSD LICENSE
	25	*
	26	* Copyright(c) 2015 Intel Corporation.
	27	*
	28	* Redistribution and use in source and binary forms, with or without
	29	* modification, are permitted provided that the following conditions
	30	* are met:
	31	*
	32	* * Redistributions of source code must retain the above copyright
	33	* notice, this list of conditions and the following disclaimer.
	34	* * Redistributions in binary form must reproduce the above copyright
	35	* notice, this list of conditions and the following disclaimer in
	36	* the documentation and/or other materials provided with the
	37	* distribution.
	38	* * Neither the name of Intel Corporation nor the names of its
	39	* contributors may be used to endorse or promote products derived
	40	* from this software without specific prior written permission.
	41	*
	42	* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
	43	* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
	44	* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
	45	* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
	46	* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
	47	* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
	48	* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
	49	* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
	50	* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
	51	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
	52	* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
	53	*
	54	*/
	55
	56	#include <linux/linkage.h>
	57
	58	#define DIGEST_PTR %rdi /* 1st arg */
	59	#define DATA_PTR %rsi /* 2nd arg */
	60	#define NUM_BLKS %rdx /* 3rd arg */
	61
	62	#define RSPSAVE %rax
	63
	64	/* gcc conversion */
	65	#define FRAME_SIZE 32 /* space for 2x16 bytes */
	66
	67	#define ABCD %xmm0
	68	#define E0 %xmm1 /* Need two E's b/c they ping pong */
	69	#define E1 %xmm2
	70	#define MSG0 %xmm3
	71	#define MSG1 %xmm4
	72	#define MSG2 %xmm5
	73	#define MSG3 %xmm6
	74	#define SHUF_MASK %xmm7
	75
	76
	77	/*
	78	* Intel SHA Extensions optimized implementation of a SHA-1 update function
	79	*
	80	* The function takes a pointer to the current hash values, a pointer to the
	81	* input data, and a number of 64 byte blocks to process. Once all blocks have
	82	* been processed, the digest pointer is updated with the resulting hash value.
	83	* The function only processes complete blocks, there is no functionality to
	84	* store partial blocks. All message padding and hash value initialization must
	85	* be done outside the update function.
	86	*
	87	* The indented lines in the loop are instructions related to rounds processing.
	88	* The non-indented lines are instructions related to the message schedule.
	89	*
	90	* void sha1_ni_transform(uint32_t digest, const void data,
	91	uint32_t numBlocks)
	92	* digest : pointer to digest
	93	* data: pointer to input data
	94	* numBlocks: Number of blocks to process
	95	*/
	96	.text
	97	.align 32
	98	ENTRY(sha1_ni_transform)
	99	mov %rsp, RSPSAVE
	100	sub $FRAME_SIZE, %rsp
	101	and $~0xF, %rsp
	102
	103	shl $6, NUM_BLKS /* convert to bytes */
	104	jz .Ldone_hash
	105	add DATA_PTR, NUM_BLKS /* pointer to end of data */
	106
	107	/* load initial hash values */
	108	pinsrd $3, 1*16(DIGEST_PTR), E0
	109	movdqu 0*16(DIGEST_PTR), ABCD
	110	pand UPPER_WORD_MASK(%rip), E0
	111	pshufd $0x1B, ABCD, ABCD
	112
	113	movdqa PSHUFFLE_BYTE_FLIP_MASK(%rip), SHUF_MASK
	114
	115	.Lloop0:
	116	/* Save hash values for addition after rounds */
	117	movdqa E0, (0*16)(%rsp)
	118	movdqa ABCD, (1*16)(%rsp)
	119
	120	/* Rounds 0-3 */
	121	movdqu 0*16(DATA_PTR), MSG0
	122	pshufb SHUF_MASK, MSG0
	123	paddd MSG0, E0
	124	movdqa ABCD, E1
	125	sha1rnds4 $0, E0, ABCD
	126
	127	/* Rounds 4-7 */
	128	movdqu 1*16(DATA_PTR), MSG1
	129	pshufb SHUF_MASK, MSG1
	130	sha1nexte MSG1, E1
	131	movdqa ABCD, E0
	132	sha1rnds4 $0, E1, ABCD
	133	sha1msg1 MSG1, MSG0
	134
	135	/* Rounds 8-11 */
	136	movdqu 2*16(DATA_PTR), MSG2
	137	pshufb SHUF_MASK, MSG2
	138	sha1nexte MSG2, E0
	139	movdqa ABCD, E1
	140	sha1rnds4 $0, E0, ABCD
	141	sha1msg1 MSG2, MSG1
	142	pxor MSG2, MSG0
	143
	144	/* Rounds 12-15 */
	145	movdqu 3*16(DATA_PTR), MSG3
	146	pshufb SHUF_MASK, MSG3
	147	sha1nexte MSG3, E1
	148	movdqa ABCD, E0
	149	sha1msg2 MSG3, MSG0
	150	sha1rnds4 $0, E1, ABCD
	151	sha1msg1 MSG3, MSG2
	152	pxor MSG3, MSG1
	153
	154	/* Rounds 16-19 */
	155	sha1nexte MSG0, E0
	156	movdqa ABCD, E1
	157	sha1msg2 MSG0, MSG1
	158	sha1rnds4 $0, E0, ABCD
	159	sha1msg1 MSG0, MSG3
	160	pxor MSG0, MSG2
	161
	162	/* Rounds 20-23 */
	163	sha1nexte MSG1, E1
	164	movdqa ABCD, E0
	165	sha1msg2 MSG1, MSG2
	166	sha1rnds4 $1, E1, ABCD
	167	sha1msg1 MSG1, MSG0
	168	pxor MSG1, MSG3
	169
	170	/* Rounds 24-27 */
	171	sha1nexte MSG2, E0
	172	movdqa ABCD, E1
	173	sha1msg2 MSG2, MSG3
	174	sha1rnds4 $1, E0, ABCD
	175	sha1msg1 MSG2, MSG1
	176	pxor MSG2, MSG0
	177
	178	/* Rounds 28-31 */
	179	sha1nexte MSG3, E1
	180	movdqa ABCD, E0
	181	sha1msg2 MSG3, MSG0
	182	sha1rnds4 $1, E1, ABCD
	183	sha1msg1 MSG3, MSG2
	184	pxor MSG3, MSG1
	185
	186	/* Rounds 32-35 */
	187	sha1nexte MSG0, E0
	188	movdqa ABCD, E1
	189	sha1msg2 MSG0, MSG1
	190	sha1rnds4 $1, E0, ABCD
	191	sha1msg1 MSG0, MSG3
	192	pxor MSG0, MSG2
	193
	194	/* Rounds 36-39 */
	195	sha1nexte MSG1, E1
	196	movdqa ABCD, E0
	197	sha1msg2 MSG1, MSG2
	198	sha1rnds4 $1, E1, ABCD
	199	sha1msg1 MSG1, MSG0
	200	pxor MSG1, MSG3
	201
	202	/* Rounds 40-43 */
	203	sha1nexte MSG2, E0
	204	movdqa ABCD, E1
	205	sha1msg2 MSG2, MSG3
	206	sha1rnds4 $2, E0, ABCD
	207	sha1msg1 MSG2, MSG1
	208	pxor MSG2, MSG0
	209
	210	/* Rounds 44-47 */
	211	sha1nexte MSG3, E1
	212	movdqa ABCD, E0
	213	sha1msg2 MSG3, MSG0
	214	sha1rnds4 $2, E1, ABCD
	215	sha1msg1 MSG3, MSG2
	216	pxor MSG3, MSG1
	217
	218	/* Rounds 48-51 */
	219	sha1nexte MSG0, E0
	220	movdqa ABCD, E1
	221	sha1msg2 MSG0, MSG1
	222	sha1rnds4 $2, E0, ABCD
	223	sha1msg1 MSG0, MSG3
	224	pxor MSG0, MSG2
	225
	226	/* Rounds 52-55 */
	227	sha1nexte MSG1, E1
	228	movdqa ABCD, E0
	229	sha1msg2 MSG1, MSG2
	230	sha1rnds4 $2, E1, ABCD
	231	sha1msg1 MSG1, MSG0
	232	pxor MSG1, MSG3
	233
	234	/* Rounds 56-59 */
	235	sha1nexte MSG2, E0
	236	movdqa ABCD, E1
	237	sha1msg2 MSG2, MSG3
	238	sha1rnds4 $2, E0, ABCD
	239	sha1msg1 MSG2, MSG1
	240	pxor MSG2, MSG0
	241
	242	/* Rounds 60-63 */
	243	sha1nexte MSG3, E1
	244	movdqa ABCD, E0
	245	sha1msg2 MSG3, MSG0
	246	sha1rnds4 $3, E1, ABCD
	247	sha1msg1 MSG3, MSG2
	248	pxor MSG3, MSG1
	249
	250	/* Rounds 64-67 */
	251	sha1nexte MSG0, E0
	252	movdqa ABCD, E1
	253	sha1msg2 MSG0, MSG1
	254	sha1rnds4 $3, E0, ABCD
	255	sha1msg1 MSG0, MSG3
	256	pxor MSG0, MSG2
	257
	258	/* Rounds 68-71 */
	259	sha1nexte MSG1, E1
	260	movdqa ABCD, E0
	261	sha1msg2 MSG1, MSG2
	262	sha1rnds4 $3, E1, ABCD
	263	pxor MSG1, MSG3
	264
	265	/* Rounds 72-75 */
	266	sha1nexte MSG2, E0
	267	movdqa ABCD, E1
	268	sha1msg2 MSG2, MSG3
	269	sha1rnds4 $3, E0, ABCD
	270
	271	/* Rounds 76-79 */
	272	sha1nexte MSG3, E1
	273	movdqa ABCD, E0
	274	sha1rnds4 $3, E1, ABCD
	275
	276	/* Add current hash values with previously saved */
	277	sha1nexte (0*16)(%rsp), E0
	278	paddd (1*16)(%rsp), ABCD
	279
	280	/* Increment data pointer and loop if more to process */
	281	add $64, DATA_PTR
	282	cmp NUM_BLKS, DATA_PTR
	283	jne .Lloop0
	284
	285	/* Write hash values back in the correct order */
	286	pshufd $0x1B, ABCD, ABCD
	287	movdqu ABCD, 0*16(DIGEST_PTR)
	288	pextrd $3, E0, 1*16(DIGEST_PTR)
	289
	290	.Ldone_hash:
	291	mov RSPSAVE, %rsp
	292
	293	ret
	294	ENDPROC(sha1_ni_transform)
	295
	296	.data
	297
	298	.align 64
	299	PSHUFFLE_BYTE_FLIP_MASK:
	300	.octa 0x000102030405060708090a0b0c0d0e0f
	301	UPPER_WORD_MASK:
	302	.octa 0xFFFFFFFF000000000000000000000000