diff options
Diffstat (limited to 'crypto/vmac.c')
| -rw-r--r-- | crypto/vmac.c | 678 |
1 files changed, 678 insertions, 0 deletions
diff --git a/crypto/vmac.c b/crypto/vmac.c new file mode 100644 index 000000000000..0a9468e575de --- /dev/null +++ b/crypto/vmac.c | |||
| @@ -0,0 +1,678 @@ | |||
| 1 | /* | ||
| 2 | * Modified to interface to the Linux kernel | ||
| 3 | * Copyright (c) 2009, Intel Corporation. | ||
| 4 | * | ||
| 5 | * This program is free software; you can redistribute it and/or modify it | ||
| 6 | * under the terms and conditions of the GNU General Public License, | ||
| 7 | * version 2, as published by the Free Software Foundation. | ||
| 8 | * | ||
| 9 | * This program is distributed in the hope it will be useful, but WITHOUT | ||
| 10 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | ||
| 11 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for | ||
| 12 | * more details. | ||
| 13 | * | ||
| 14 | * You should have received a copy of the GNU General Public License along with | ||
| 15 | * this program; if not, write to the Free Software Foundation, Inc., 59 Temple | ||
| 16 | * Place - Suite 330, Boston, MA 02111-1307 USA. | ||
| 17 | */ | ||
| 18 | |||
| 19 | /* -------------------------------------------------------------------------- | ||
| 20 | * VMAC and VHASH Implementation by Ted Krovetz (tdk@acm.org) and Wei Dai. | ||
| 21 | * This implementation is herby placed in the public domain. | ||
| 22 | * The authors offers no warranty. Use at your own risk. | ||
| 23 | * Please send bug reports to the authors. | ||
| 24 | * Last modified: 17 APR 08, 1700 PDT | ||
| 25 | * ----------------------------------------------------------------------- */ | ||
| 26 | |||
| 27 | #include <linux/init.h> | ||
| 28 | #include <linux/types.h> | ||
| 29 | #include <linux/crypto.h> | ||
| 30 | #include <linux/scatterlist.h> | ||
| 31 | #include <asm/byteorder.h> | ||
| 32 | #include <crypto/scatterwalk.h> | ||
| 33 | #include <crypto/vmac.h> | ||
| 34 | #include <crypto/internal/hash.h> | ||
| 35 | |||
| 36 | /* | ||
| 37 | * Constants and masks | ||
| 38 | */ | ||
| 39 | #define UINT64_C(x) x##ULL | ||
| 40 | const u64 p64 = UINT64_C(0xfffffffffffffeff); /* 2^64 - 257 prime */ | ||
| 41 | const u64 m62 = UINT64_C(0x3fffffffffffffff); /* 62-bit mask */ | ||
| 42 | const u64 m63 = UINT64_C(0x7fffffffffffffff); /* 63-bit mask */ | ||
| 43 | const u64 m64 = UINT64_C(0xffffffffffffffff); /* 64-bit mask */ | ||
| 44 | const u64 mpoly = UINT64_C(0x1fffffff1fffffff); /* Poly key mask */ | ||
| 45 | |||
| 46 | #ifdef __LITTLE_ENDIAN | ||
| 47 | #define INDEX_HIGH 1 | ||
| 48 | #define INDEX_LOW 0 | ||
| 49 | #else | ||
| 50 | #define INDEX_HIGH 0 | ||
| 51 | #define INDEX_LOW 1 | ||
| 52 | #endif | ||
| 53 | |||
| 54 | /* | ||
| 55 | * The following routines are used in this implementation. They are | ||
| 56 | * written via macros to simulate zero-overhead call-by-reference. | ||
| 57 | * | ||
| 58 | * MUL64: 64x64->128-bit multiplication | ||
| 59 | * PMUL64: assumes top bits cleared on inputs | ||
| 60 | * ADD128: 128x128->128-bit addition | ||
| 61 | */ | ||
| 62 | |||
| 63 | #define ADD128(rh, rl, ih, il) \ | ||
| 64 | do { \ | ||
| 65 | u64 _il = (il); \ | ||
| 66 | (rl) += (_il); \ | ||
| 67 | if ((rl) < (_il)) \ | ||
| 68 | (rh)++; \ | ||
| 69 | (rh) += (ih); \ | ||
| 70 | } while (0) | ||
| 71 | |||
| 72 | #define MUL32(i1, i2) ((u64)(u32)(i1)*(u32)(i2)) | ||
| 73 | |||
| 74 | #define PMUL64(rh, rl, i1, i2) /* Assumes m doesn't overflow */ \ | ||
| 75 | do { \ | ||
| 76 | u64 _i1 = (i1), _i2 = (i2); \ | ||
| 77 | u64 m = MUL32(_i1, _i2>>32) + MUL32(_i1>>32, _i2); \ | ||
| 78 | rh = MUL32(_i1>>32, _i2>>32); \ | ||
| 79 | rl = MUL32(_i1, _i2); \ | ||
| 80 | ADD128(rh, rl, (m >> 32), (m << 32)); \ | ||
| 81 | } while (0) | ||
| 82 | |||
| 83 | #define MUL64(rh, rl, i1, i2) \ | ||
| 84 | do { \ | ||
| 85 | u64 _i1 = (i1), _i2 = (i2); \ | ||
| 86 | u64 m1 = MUL32(_i1, _i2>>32); \ | ||
| 87 | u64 m2 = MUL32(_i1>>32, _i2); \ | ||
| 88 | rh = MUL32(_i1>>32, _i2>>32); \ | ||
| 89 | rl = MUL32(_i1, _i2); \ | ||
| 90 | ADD128(rh, rl, (m1 >> 32), (m1 << 32)); \ | ||
| 91 | ADD128(rh, rl, (m2 >> 32), (m2 << 32)); \ | ||
| 92 | } while (0) | ||
| 93 | |||
| 94 | /* | ||
| 95 | * For highest performance the L1 NH and L2 polynomial hashes should be | ||
| 96 | * carefully implemented to take advantage of one's target architechture. | ||
| 97 | * Here these two hash functions are defined multiple time; once for | ||
| 98 | * 64-bit architectures, once for 32-bit SSE2 architectures, and once | ||
| 99 | * for the rest (32-bit) architectures. | ||
| 100 | * For each, nh_16 *must* be defined (works on multiples of 16 bytes). | ||
| 101 | * Optionally, nh_vmac_nhbytes can be defined (for multiples of | ||
| 102 | * VMAC_NHBYTES), and nh_16_2 and nh_vmac_nhbytes_2 (versions that do two | ||
| 103 | * NH computations at once). | ||
| 104 | */ | ||
| 105 | |||
| 106 | #ifdef CONFIG_64BIT | ||
| 107 | |||
| 108 | #define nh_16(mp, kp, nw, rh, rl) \ | ||
| 109 | do { \ | ||
| 110 | int i; u64 th, tl; \ | ||
| 111 | rh = rl = 0; \ | ||
| 112 | for (i = 0; i < nw; i += 2) { \ | ||
| 113 | MUL64(th, tl, le64_to_cpup((mp)+i)+(kp)[i], \ | ||
| 114 | le64_to_cpup((mp)+i+1)+(kp)[i+1]); \ | ||
| 115 | ADD128(rh, rl, th, tl); \ | ||
| 116 | } \ | ||
| 117 | } while (0) | ||
| 118 | |||
| 119 | #define nh_16_2(mp, kp, nw, rh, rl, rh1, rl1) \ | ||
| 120 | do { \ | ||
| 121 | int i; u64 th, tl; \ | ||
| 122 | rh1 = rl1 = rh = rl = 0; \ | ||
| 123 | for (i = 0; i < nw; i += 2) { \ | ||
| 124 | MUL64(th, tl, le64_to_cpup((mp)+i)+(kp)[i], \ | ||
| 125 | le64_to_cpup((mp)+i+1)+(kp)[i+1]); \ | ||
| 126 | ADD128(rh, rl, th, tl); \ | ||
| 127 | MUL64(th, tl, le64_to_cpup((mp)+i)+(kp)[i+2], \ | ||
| 128 | le64_to_cpup((mp)+i+1)+(kp)[i+3]); \ | ||
| 129 | ADD128(rh1, rl1, th, tl); \ | ||
| 130 | } \ | ||
| 131 | } while (0) | ||
| 132 | |||
| 133 | #if (VMAC_NHBYTES >= 64) /* These versions do 64-bytes of message at a time */ | ||
| 134 | #define nh_vmac_nhbytes(mp, kp, nw, rh, rl) \ | ||
| 135 | do { \ | ||
| 136 | int i; u64 th, tl; \ | ||
| 137 | rh = rl = 0; \ | ||
| 138 | for (i = 0; i < nw; i += 8) { \ | ||
| 139 | MUL64(th, tl, le64_to_cpup((mp)+i)+(kp)[i], \ | ||
| 140 | le64_to_cpup((mp)+i+1)+(kp)[i+1]); \ | ||
| 141 | ADD128(rh, rl, th, tl); \ | ||
| 142 | MUL64(th, tl, le64_to_cpup((mp)+i+2)+(kp)[i+2], \ | ||
| 143 | le64_to_cpup((mp)+i+3)+(kp)[i+3]); \ | ||
| 144 | ADD128(rh, rl, th, tl); \ | ||
| 145 | MUL64(th, tl, le64_to_cpup((mp)+i+4)+(kp)[i+4], \ | ||
| 146 | le64_to_cpup((mp)+i+5)+(kp)[i+5]); \ | ||
| 147 | ADD128(rh, rl, th, tl); \ | ||
| 148 | MUL64(th, tl, le64_to_cpup((mp)+i+6)+(kp)[i+6], \ | ||
| 149 | le64_to_cpup((mp)+i+7)+(kp)[i+7]); \ | ||
| 150 | ADD128(rh, rl, th, tl); \ | ||
| 151 | } \ | ||
| 152 | } while (0) | ||
| 153 | |||
| 154 | #define nh_vmac_nhbytes_2(mp, kp, nw, rh, rl, rh1, rl1) \ | ||
| 155 | do { \ | ||
| 156 | int i; u64 th, tl; \ | ||
| 157 | rh1 = rl1 = rh = rl = 0; \ | ||
| 158 | for (i = 0; i < nw; i += 8) { \ | ||
| 159 | MUL64(th, tl, le64_to_cpup((mp)+i)+(kp)[i], \ | ||
| 160 | le64_to_cpup((mp)+i+1)+(kp)[i+1]); \ | ||
| 161 | ADD128(rh, rl, th, tl); \ | ||
| 162 | MUL64(th, tl, le64_to_cpup((mp)+i)+(kp)[i+2], \ | ||
| 163 | le64_to_cpup((mp)+i+1)+(kp)[i+3]); \ | ||
| 164 | ADD128(rh1, rl1, th, tl); \ | ||
| 165 | MUL64(th, tl, le64_to_cpup((mp)+i+2)+(kp)[i+2], \ | ||
| 166 | le64_to_cpup((mp)+i+3)+(kp)[i+3]); \ | ||
| 167 | ADD128(rh, rl, th, tl); \ | ||
| 168 | MUL64(th, tl, le64_to_cpup((mp)+i+2)+(kp)[i+4], \ | ||
| 169 | le64_to_cpup((mp)+i+3)+(kp)[i+5]); \ | ||
| 170 | ADD128(rh1, rl1, th, tl); \ | ||
| 171 | MUL64(th, tl, le64_to_cpup((mp)+i+4)+(kp)[i+4], \ | ||
| 172 | le64_to_cpup((mp)+i+5)+(kp)[i+5]); \ | ||
| 173 | ADD128(rh, rl, th, tl); \ | ||
| 174 | MUL64(th, tl, le64_to_cpup((mp)+i+4)+(kp)[i+6], \ | ||
| 175 | le64_to_cpup((mp)+i+5)+(kp)[i+7]); \ | ||
| 176 | ADD128(rh1, rl1, th, tl); \ | ||
| 177 | MUL64(th, tl, le64_to_cpup((mp)+i+6)+(kp)[i+6], \ | ||
| 178 | le64_to_cpup((mp)+i+7)+(kp)[i+7]); \ | ||
| 179 | ADD128(rh, rl, th, tl); \ | ||
| 180 | MUL64(th, tl, le64_to_cpup((mp)+i+6)+(kp)[i+8], \ | ||
| 181 | le64_to_cpup((mp)+i+7)+(kp)[i+9]); \ | ||
| 182 | ADD128(rh1, rl1, th, tl); \ | ||
| 183 | } \ | ||
| 184 | } while (0) | ||
| 185 | #endif | ||
| 186 | |||
| 187 | #define poly_step(ah, al, kh, kl, mh, ml) \ | ||
| 188 | do { \ | ||
| 189 | u64 t1h, t1l, t2h, t2l, t3h, t3l, z = 0; \ | ||
| 190 | /* compute ab*cd, put bd into result registers */ \ | ||
| 191 | PMUL64(t3h, t3l, al, kh); \ | ||
| 192 | PMUL64(t2h, t2l, ah, kl); \ | ||
| 193 | PMUL64(t1h, t1l, ah, 2*kh); \ | ||
| 194 | PMUL64(ah, al, al, kl); \ | ||
| 195 | /* add 2 * ac to result */ \ | ||
| 196 | ADD128(ah, al, t1h, t1l); \ | ||
| 197 | /* add together ad + bc */ \ | ||
| 198 | ADD128(t2h, t2l, t3h, t3l); \ | ||
| 199 | /* now (ah,al), (t2l,2*t2h) need summing */ \ | ||
| 200 | /* first add the high registers, carrying into t2h */ \ | ||
| 201 | ADD128(t2h, ah, z, t2l); \ | ||
| 202 | /* double t2h and add top bit of ah */ \ | ||
| 203 | t2h = 2 * t2h + (ah >> 63); \ | ||
| 204 | ah &= m63; \ | ||
| 205 | /* now add the low registers */ \ | ||
| 206 | ADD128(ah, al, mh, ml); \ | ||
| 207 | ADD128(ah, al, z, t2h); \ | ||
| 208 | } while (0) | ||
| 209 | |||
| 210 | #else /* ! CONFIG_64BIT */ | ||
| 211 | |||
| 212 | #ifndef nh_16 | ||
| 213 | #define nh_16(mp, kp, nw, rh, rl) \ | ||
| 214 | do { \ | ||
| 215 | u64 t1, t2, m1, m2, t; \ | ||
| 216 | int i; \ | ||
| 217 | rh = rl = t = 0; \ | ||
| 218 | for (i = 0; i < nw; i += 2) { \ | ||
| 219 | t1 = le64_to_cpup(mp+i) + kp[i]; \ | ||
| 220 | t2 = le64_to_cpup(mp+i+1) + kp[i+1]; \ | ||
| 221 | m2 = MUL32(t1 >> 32, t2); \ | ||
| 222 | m1 = MUL32(t1, t2 >> 32); \ | ||
| 223 | ADD128(rh, rl, MUL32(t1 >> 32, t2 >> 32), \ | ||
| 224 | MUL32(t1, t2)); \ | ||
| 225 | rh += (u64)(u32)(m1 >> 32) \ | ||
| 226 | + (u32)(m2 >> 32); \ | ||
| 227 | t += (u64)(u32)m1 + (u32)m2; \ | ||
| 228 | } \ | ||
| 229 | ADD128(rh, rl, (t >> 32), (t << 32)); \ | ||
| 230 | } while (0) | ||
| 231 | #endif | ||
| 232 | |||
| 233 | static void poly_step_func(u64 *ahi, u64 *alo, | ||
| 234 | const u64 *kh, const u64 *kl, | ||
| 235 | const u64 *mh, const u64 *ml) | ||
| 236 | { | ||
| 237 | #define a0 (*(((u32 *)alo)+INDEX_LOW)) | ||
| 238 | #define a1 (*(((u32 *)alo)+INDEX_HIGH)) | ||
| 239 | #define a2 (*(((u32 *)ahi)+INDEX_LOW)) | ||
| 240 | #define a3 (*(((u32 *)ahi)+INDEX_HIGH)) | ||
| 241 | #define k0 (*(((u32 *)kl)+INDEX_LOW)) | ||
| 242 | #define k1 (*(((u32 *)kl)+INDEX_HIGH)) | ||
| 243 | #define k2 (*(((u32 *)kh)+INDEX_LOW)) | ||
| 244 | #define k3 (*(((u32 *)kh)+INDEX_HIGH)) | ||
| 245 | |||
| 246 | u64 p, q, t; | ||
| 247 | u32 t2; | ||
| 248 | |||
| 249 | p = MUL32(a3, k3); | ||
| 250 | p += p; | ||
| 251 | p += *(u64 *)mh; | ||
| 252 | p += MUL32(a0, k2); | ||
| 253 | p += MUL32(a1, k1); | ||
| 254 | p += MUL32(a2, k0); | ||
| 255 | t = (u32)(p); | ||
| 256 | p >>= 32; | ||
| 257 | p += MUL32(a0, k3); | ||
| 258 | p += MUL32(a1, k2); | ||
| 259 | p += MUL32(a2, k1); | ||
| 260 | p += MUL32(a3, k0); | ||
| 261 | t |= ((u64)((u32)p & 0x7fffffff)) << 32; | ||
| 262 | p >>= 31; | ||
| 263 | p += (u64)(((u32 *)ml)[INDEX_LOW]); | ||
| 264 | p += MUL32(a0, k0); | ||
| 265 | q = MUL32(a1, k3); | ||
| 266 | q += MUL32(a2, k2); | ||
| 267 | q += MUL32(a3, k1); | ||
| 268 | q += q; | ||
| 269 | p += q; | ||
| 270 | t2 = (u32)(p); | ||
| 271 | p >>= 32; | ||
| 272 | p += (u64)(((u32 *)ml)[INDEX_HIGH]); | ||
| 273 | p += MUL32(a0, k1); | ||
| 274 | p += MUL32(a1, k0); | ||
| 275 | q = MUL32(a2, k3); | ||
| 276 | q += MUL32(a3, k2); | ||
| 277 | q += q; | ||
| 278 | p += q; | ||
| 279 | *(u64 *)(alo) = (p << 32) | t2; | ||
| 280 | p >>= 32; | ||
| 281 | *(u64 *)(ahi) = p + t; | ||
| 282 | |||
| 283 | #undef a0 | ||
| 284 | #undef a1 | ||
| 285 | #undef a2 | ||
| 286 | #undef a3 | ||
| 287 | #undef k0 | ||
| 288 | #undef k1 | ||
| 289 | #undef k2 | ||
| 290 | #undef k3 | ||
| 291 | } | ||
| 292 | |||
| 293 | #define poly_step(ah, al, kh, kl, mh, ml) \ | ||
| 294 | poly_step_func(&(ah), &(al), &(kh), &(kl), &(mh), &(ml)) | ||
| 295 | |||
| 296 | #endif /* end of specialized NH and poly definitions */ | ||
| 297 | |||
| 298 | /* At least nh_16 is defined. Defined others as needed here */ | ||
| 299 | #ifndef nh_16_2 | ||
| 300 | #define nh_16_2(mp, kp, nw, rh, rl, rh2, rl2) \ | ||
| 301 | do { \ | ||
| 302 | nh_16(mp, kp, nw, rh, rl); \ | ||
| 303 | nh_16(mp, ((kp)+2), nw, rh2, rl2); \ | ||
| 304 | } while (0) | ||
| 305 | #endif | ||
| 306 | #ifndef nh_vmac_nhbytes | ||
| 307 | #define nh_vmac_nhbytes(mp, kp, nw, rh, rl) \ | ||
| 308 | nh_16(mp, kp, nw, rh, rl) | ||
| 309 | #endif | ||
| 310 | #ifndef nh_vmac_nhbytes_2 | ||
| 311 | #define nh_vmac_nhbytes_2(mp, kp, nw, rh, rl, rh2, rl2) \ | ||
| 312 | do { \ | ||
| 313 | nh_vmac_nhbytes(mp, kp, nw, rh, rl); \ | ||
| 314 | nh_vmac_nhbytes(mp, ((kp)+2), nw, rh2, rl2); \ | ||
| 315 | } while (0) | ||
| 316 | #endif | ||
| 317 | |||
| 318 | static void vhash_abort(struct vmac_ctx *ctx) | ||
| 319 | { | ||
| 320 | ctx->polytmp[0] = ctx->polykey[0] ; | ||
| 321 | ctx->polytmp[1] = ctx->polykey[1] ; | ||
| 322 | ctx->first_block_processed = 0; | ||
| 323 | } | ||
| 324 | |||
| 325 | static u64 l3hash(u64 p1, u64 p2, | ||
| 326 | u64 k1, u64 k2, u64 len) | ||
| 327 | { | ||
| 328 | u64 rh, rl, t, z = 0; | ||
| 329 | |||
| 330 | /* fully reduce (p1,p2)+(len,0) mod p127 */ | ||
| 331 | t = p1 >> 63; | ||
| 332 | p1 &= m63; | ||
| 333 | ADD128(p1, p2, len, t); | ||
| 334 | /* At this point, (p1,p2) is at most 2^127+(len<<64) */ | ||
| 335 | t = (p1 > m63) + ((p1 == m63) && (p2 == m64)); | ||
| 336 | ADD128(p1, p2, z, t); | ||
| 337 | p1 &= m63; | ||
| 338 | |||
| 339 | /* compute (p1,p2)/(2^64-2^32) and (p1,p2)%(2^64-2^32) */ | ||
| 340 | t = p1 + (p2 >> 32); | ||
| 341 | t += (t >> 32); | ||
| 342 | t += (u32)t > 0xfffffffeu; | ||
| 343 | p1 += (t >> 32); | ||
| 344 | p2 += (p1 << 32); | ||
| 345 | |||
| 346 | /* compute (p1+k1)%p64 and (p2+k2)%p64 */ | ||
| 347 | p1 += k1; | ||
| 348 | p1 += (0 - (p1 < k1)) & 257; | ||
| 349 | p2 += k2; | ||
| 350 | p2 += (0 - (p2 < k2)) & 257; | ||
| 351 | |||
| 352 | /* compute (p1+k1)*(p2+k2)%p64 */ | ||
| 353 | MUL64(rh, rl, p1, p2); | ||
| 354 | t = rh >> 56; | ||
| 355 | ADD128(t, rl, z, rh); | ||
| 356 | rh <<= 8; | ||
| 357 | ADD128(t, rl, z, rh); | ||
| 358 | t += t << 8; | ||
| 359 | rl += t; | ||
| 360 | rl += (0 - (rl < t)) & 257; | ||
| 361 | rl += (0 - (rl > p64-1)) & 257; | ||
| 362 | return rl; | ||
| 363 | } | ||
| 364 | |||
| 365 | static void vhash_update(const unsigned char *m, | ||
| 366 | unsigned int mbytes, /* Pos multiple of VMAC_NHBYTES */ | ||
| 367 | struct vmac_ctx *ctx) | ||
| 368 | { | ||
| 369 | u64 rh, rl, *mptr; | ||
| 370 | const u64 *kptr = (u64 *)ctx->nhkey; | ||
| 371 | int i; | ||
| 372 | u64 ch, cl; | ||
| 373 | u64 pkh = ctx->polykey[0]; | ||
| 374 | u64 pkl = ctx->polykey[1]; | ||
| 375 | |||
| 376 | mptr = (u64 *)m; | ||
| 377 | i = mbytes / VMAC_NHBYTES; /* Must be non-zero */ | ||
| 378 | |||
| 379 | ch = ctx->polytmp[0]; | ||
| 380 | cl = ctx->polytmp[1]; | ||
| 381 | |||
| 382 | if (!ctx->first_block_processed) { | ||
| 383 | ctx->first_block_processed = 1; | ||
| 384 | nh_vmac_nhbytes(mptr, kptr, VMAC_NHBYTES/8, rh, rl); | ||
| 385 | rh &= m62; | ||
| 386 | ADD128(ch, cl, rh, rl); | ||
| 387 | mptr += (VMAC_NHBYTES/sizeof(u64)); | ||
| 388 | i--; | ||
| 389 | } | ||
| 390 | |||
| 391 | while (i--) { | ||
| 392 | nh_vmac_nhbytes(mptr, kptr, VMAC_NHBYTES/8, rh, rl); | ||
| 393 | rh &= m62; | ||
| 394 | poly_step(ch, cl, pkh, pkl, rh, rl); | ||
| 395 | mptr += (VMAC_NHBYTES/sizeof(u64)); | ||
| 396 | } | ||
| 397 | |||
| 398 | ctx->polytmp[0] = ch; | ||
| 399 | ctx->polytmp[1] = cl; | ||
| 400 | } | ||
| 401 | |||
| 402 | static u64 vhash(unsigned char m[], unsigned int mbytes, | ||
| 403 | u64 *tagl, struct vmac_ctx *ctx) | ||
| 404 | { | ||
| 405 | u64 rh, rl, *mptr; | ||
| 406 | const u64 *kptr = (u64 *)ctx->nhkey; | ||
| 407 | int i, remaining; | ||
| 408 | u64 ch, cl; | ||
| 409 | u64 pkh = ctx->polykey[0]; | ||
| 410 | u64 pkl = ctx->polykey[1]; | ||
| 411 | |||
| 412 | mptr = (u64 *)m; | ||
| 413 | i = mbytes / VMAC_NHBYTES; | ||
| 414 | remaining = mbytes % VMAC_NHBYTES; | ||
| 415 | |||
| 416 | if (ctx->first_block_processed) { | ||
| 417 | ch = ctx->polytmp[0]; | ||
| 418 | cl = ctx->polytmp[1]; | ||
| 419 | } else if (i) { | ||
| 420 | nh_vmac_nhbytes(mptr, kptr, VMAC_NHBYTES/8, ch, cl); | ||
| 421 | ch &= m62; | ||
| 422 | ADD128(ch, cl, pkh, pkl); | ||
| 423 | mptr += (VMAC_NHBYTES/sizeof(u64)); | ||
| 424 | i--; | ||
| 425 | } else if (remaining) { | ||
| 426 | nh_16(mptr, kptr, 2*((remaining+15)/16), ch, cl); | ||
| 427 | ch &= m62; | ||
| 428 | ADD128(ch, cl, pkh, pkl); | ||
| 429 | mptr += (VMAC_NHBYTES/sizeof(u64)); | ||
| 430 | goto do_l3; | ||
| 431 | } else {/* Empty String */ | ||
| 432 | ch = pkh; cl = pkl; | ||
| 433 | goto do_l3; | ||
| 434 | } | ||
| 435 | |||
| 436 | while (i--) { | ||
| 437 | nh_vmac_nhbytes(mptr, kptr, VMAC_NHBYTES/8, rh, rl); | ||
| 438 | rh &= m62; | ||
| 439 | poly_step(ch, cl, pkh, pkl, rh, rl); | ||
| 440 | mptr += (VMAC_NHBYTES/sizeof(u64)); | ||
| 441 | } | ||
| 442 | if (remaining) { | ||
| 443 | nh_16(mptr, kptr, 2*((remaining+15)/16), rh, rl); | ||
| 444 | rh &= m62; | ||
| 445 | poly_step(ch, cl, pkh, pkl, rh, rl); | ||
| 446 | } | ||
| 447 | |||
| 448 | do_l3: | ||
| 449 | vhash_abort(ctx); | ||
| 450 | remaining *= 8; | ||
| 451 | return l3hash(ch, cl, ctx->l3key[0], ctx->l3key[1], remaining); | ||
| 452 | } | ||
| 453 | |||
| 454 | static u64 vmac(unsigned char m[], unsigned int mbytes, | ||
| 455 | unsigned char n[16], u64 *tagl, | ||
| 456 | struct vmac_ctx_t *ctx) | ||
| 457 | { | ||
| 458 | u64 *in_n, *out_p; | ||
| 459 | u64 p, h; | ||
| 460 | int i; | ||
| 461 | |||
| 462 | in_n = ctx->__vmac_ctx.cached_nonce; | ||
| 463 | out_p = ctx->__vmac_ctx.cached_aes; | ||
| 464 | |||
| 465 | i = n[15] & 1; | ||
| 466 | if ((*(u64 *)(n+8) != in_n[1]) || (*(u64 *)(n) != in_n[0])) { | ||
| 467 | in_n[0] = *(u64 *)(n); | ||
| 468 | in_n[1] = *(u64 *)(n+8); | ||
| 469 | ((unsigned char *)in_n)[15] &= 0xFE; | ||
| 470 | crypto_cipher_encrypt_one(ctx->child, | ||
| 471 | (unsigned char *)out_p, (unsigned char *)in_n); | ||
| 472 | |||
| 473 | ((unsigned char *)in_n)[15] |= (unsigned char)(1-i); | ||
| 474 | } | ||
| 475 | p = be64_to_cpup(out_p + i); | ||
| 476 | h = vhash(m, mbytes, (u64 *)0, &ctx->__vmac_ctx); | ||
| 477 | return p + h; | ||
| 478 | } | ||
| 479 | |||
| 480 | static int vmac_set_key(unsigned char user_key[], struct vmac_ctx_t *ctx) | ||
| 481 | { | ||
| 482 | u64 in[2] = {0}, out[2]; | ||
| 483 | unsigned i; | ||
| 484 | int err = 0; | ||
| 485 | |||
| 486 | err = crypto_cipher_setkey(ctx->child, user_key, VMAC_KEY_LEN); | ||
| 487 | if (err) | ||
| 488 | return err; | ||
| 489 | |||
| 490 | /* Fill nh key */ | ||
| 491 | ((unsigned char *)in)[0] = 0x80; | ||
| 492 | for (i = 0; i < sizeof(ctx->__vmac_ctx.nhkey)/8; i += 2) { | ||
| 493 | crypto_cipher_encrypt_one(ctx->child, | ||
| 494 | (unsigned char *)out, (unsigned char *)in); | ||
| 495 | ctx->__vmac_ctx.nhkey[i] = be64_to_cpup(out); | ||
| 496 | ctx->__vmac_ctx.nhkey[i+1] = be64_to_cpup(out+1); | ||
| 497 | ((unsigned char *)in)[15] += 1; | ||
| 498 | } | ||
| 499 | |||
| 500 | /* Fill poly key */ | ||
| 501 | ((unsigned char *)in)[0] = 0xC0; | ||
| 502 | in[1] = 0; | ||
| 503 | for (i = 0; i < sizeof(ctx->__vmac_ctx.polykey)/8; i += 2) { | ||
| 504 | crypto_cipher_encrypt_one(ctx->child, | ||
| 505 | (unsigned char *)out, (unsigned char *)in); | ||
| 506 | ctx->__vmac_ctx.polytmp[i] = | ||
| 507 | ctx->__vmac_ctx.polykey[i] = | ||
| 508 | be64_to_cpup(out) & mpoly; | ||
| 509 | ctx->__vmac_ctx.polytmp[i+1] = | ||
| 510 | ctx->__vmac_ctx.polykey[i+1] = | ||
| 511 | be64_to_cpup(out+1) & mpoly; | ||
| 512 | ((unsigned char *)in)[15] += 1; | ||
| 513 | } | ||
| 514 | |||
| 515 | /* Fill ip key */ | ||
| 516 | ((unsigned char *)in)[0] = 0xE0; | ||
| 517 | in[1] = 0; | ||
| 518 | for (i = 0; i < sizeof(ctx->__vmac_ctx.l3key)/8; i += 2) { | ||
| 519 | do { | ||
| 520 | crypto_cipher_encrypt_one(ctx->child, | ||
| 521 | (unsigned char *)out, (unsigned char *)in); | ||
| 522 | ctx->__vmac_ctx.l3key[i] = be64_to_cpup(out); | ||
| 523 | ctx->__vmac_ctx.l3key[i+1] = be64_to_cpup(out+1); | ||
| 524 | ((unsigned char *)in)[15] += 1; | ||
| 525 | } while (ctx->__vmac_ctx.l3key[i] >= p64 | ||
| 526 | || ctx->__vmac_ctx.l3key[i+1] >= p64); | ||
| 527 | } | ||
| 528 | |||
| 529 | /* Invalidate nonce/aes cache and reset other elements */ | ||
| 530 | ctx->__vmac_ctx.cached_nonce[0] = (u64)-1; /* Ensure illegal nonce */ | ||
| 531 | ctx->__vmac_ctx.cached_nonce[1] = (u64)0; /* Ensure illegal nonce */ | ||
| 532 | ctx->__vmac_ctx.first_block_processed = 0; | ||
| 533 | |||
| 534 | return err; | ||
| 535 | } | ||
| 536 | |||
| 537 | static int vmac_setkey(struct crypto_shash *parent, | ||
| 538 | const u8 *key, unsigned int keylen) | ||
| 539 | { | ||
| 540 | struct vmac_ctx_t *ctx = crypto_shash_ctx(parent); | ||
| 541 | |||
| 542 | if (keylen != VMAC_KEY_LEN) { | ||
| 543 | crypto_shash_set_flags(parent, CRYPTO_TFM_RES_BAD_KEY_LEN); | ||
| 544 | return -EINVAL; | ||
| 545 | } | ||
| 546 | |||
| 547 | return vmac_set_key((u8 *)key, ctx); | ||
| 548 | } | ||
| 549 | |||
| 550 | static int vmac_init(struct shash_desc *pdesc) | ||
| 551 | { | ||
| 552 | struct crypto_shash *parent = pdesc->tfm; | ||
| 553 | struct vmac_ctx_t *ctx = crypto_shash_ctx(parent); | ||
| 554 | |||
| 555 | memset(&ctx->__vmac_ctx, 0, sizeof(struct vmac_ctx)); | ||
| 556 | return 0; | ||
| 557 | } | ||
| 558 | |||
| 559 | static int vmac_update(struct shash_desc *pdesc, const u8 *p, | ||
| 560 | unsigned int len) | ||
| 561 | { | ||
| 562 | struct crypto_shash *parent = pdesc->tfm; | ||
| 563 | struct vmac_ctx_t *ctx = crypto_shash_ctx(parent); | ||
| 564 | |||
| 565 | vhash_update(p, len, &ctx->__vmac_ctx); | ||
| 566 | |||
| 567 | return 0; | ||
| 568 | } | ||
| 569 | |||
| 570 | static int vmac_final(struct shash_desc *pdesc, u8 *out) | ||
| 571 | { | ||
| 572 | struct crypto_shash *parent = pdesc->tfm; | ||
| 573 | struct vmac_ctx_t *ctx = crypto_shash_ctx(parent); | ||
| 574 | vmac_t mac; | ||
| 575 | u8 nonce[16] = {}; | ||
| 576 | |||
| 577 | mac = vmac(NULL, 0, nonce, NULL, ctx); | ||
| 578 | memcpy(out, &mac, sizeof(vmac_t)); | ||
| 579 | memset(&mac, 0, sizeof(vmac_t)); | ||
| 580 | memset(&ctx->__vmac_ctx, 0, sizeof(struct vmac_ctx)); | ||
| 581 | return 0; | ||
| 582 | } | ||
| 583 | |||
| 584 | static int vmac_init_tfm(struct crypto_tfm *tfm) | ||
| 585 | { | ||
| 586 | struct crypto_cipher *cipher; | ||
| 587 | struct crypto_instance *inst = (void *)tfm->__crt_alg; | ||
| 588 | struct crypto_spawn *spawn = crypto_instance_ctx(inst); | ||
| 589 | struct vmac_ctx_t *ctx = crypto_tfm_ctx(tfm); | ||
| 590 | |||
| 591 | cipher = crypto_spawn_cipher(spawn); | ||
| 592 | if (IS_ERR(cipher)) | ||
| 593 | return PTR_ERR(cipher); | ||
| 594 | |||
| 595 | ctx->child = cipher; | ||
| 596 | return 0; | ||
| 597 | } | ||
| 598 | |||
| 599 | static void vmac_exit_tfm(struct crypto_tfm *tfm) | ||
| 600 | { | ||
| 601 | struct vmac_ctx_t *ctx = crypto_tfm_ctx(tfm); | ||
| 602 | crypto_free_cipher(ctx->child); | ||
| 603 | } | ||
| 604 | |||
| 605 | static int vmac_create(struct crypto_template *tmpl, struct rtattr **tb) | ||
| 606 | { | ||
| 607 | struct shash_instance *inst; | ||
| 608 | struct crypto_alg *alg; | ||
| 609 | int err; | ||
| 610 | |||
| 611 | err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_SHASH); | ||
| 612 | if (err) | ||
| 613 | return err; | ||
| 614 | |||
| 615 | alg = crypto_get_attr_alg(tb, CRYPTO_ALG_TYPE_CIPHER, | ||
| 616 | CRYPTO_ALG_TYPE_MASK); | ||
| 617 | if (IS_ERR(alg)) | ||
| 618 | return PTR_ERR(alg); | ||
| 619 | |||
| 620 | inst = shash_alloc_instance("vmac", alg); | ||
| 621 | err = PTR_ERR(inst); | ||
| 622 | if (IS_ERR(inst)) | ||
| 623 | goto out_put_alg; | ||
| 624 | |||
| 625 | err = crypto_init_spawn(shash_instance_ctx(inst), alg, | ||
| 626 | shash_crypto_instance(inst), | ||
| 627 | CRYPTO_ALG_TYPE_MASK); | ||
| 628 | if (err) | ||
| 629 | goto out_free_inst; | ||
| 630 | |||
| 631 | inst->alg.base.cra_priority = alg->cra_priority; | ||
| 632 | inst->alg.base.cra_blocksize = alg->cra_blocksize; | ||
| 633 | inst->alg.base.cra_alignmask = alg->cra_alignmask; | ||
| 634 | |||
| 635 | inst->alg.digestsize = sizeof(vmac_t); | ||
| 636 | inst->alg.base.cra_ctxsize = sizeof(struct vmac_ctx_t); | ||
| 637 | inst->alg.base.cra_init = vmac_init_tfm; | ||
| 638 | inst->alg.base.cra_exit = vmac_exit_tfm; | ||
| 639 | |||
| 640 | inst->alg.init = vmac_init; | ||
| 641 | inst->alg.update = vmac_update; | ||
| 642 | inst->alg.final = vmac_final; | ||
| 643 | inst->alg.setkey = vmac_setkey; | ||
| 644 | |||
| 645 | err = shash_register_instance(tmpl, inst); | ||
| 646 | if (err) { | ||
| 647 | out_free_inst: | ||
| 648 | shash_free_instance(shash_crypto_instance(inst)); | ||
| 649 | } | ||
| 650 | |||
| 651 | out_put_alg: | ||
| 652 | crypto_mod_put(alg); | ||
| 653 | return err; | ||
| 654 | } | ||
| 655 | |||
| 656 | static struct crypto_template vmac_tmpl = { | ||
| 657 | .name = "vmac", | ||
| 658 | .create = vmac_create, | ||
| 659 | .free = shash_free_instance, | ||
| 660 | .module = THIS_MODULE, | ||
| 661 | }; | ||
| 662 | |||
| 663 | static int __init vmac_module_init(void) | ||
| 664 | { | ||
| 665 | return crypto_register_template(&vmac_tmpl); | ||
| 666 | } | ||
| 667 | |||
| 668 | static void __exit vmac_module_exit(void) | ||
| 669 | { | ||
| 670 | crypto_unregister_template(&vmac_tmpl); | ||
| 671 | } | ||
| 672 | |||
| 673 | module_init(vmac_module_init); | ||
| 674 | module_exit(vmac_module_exit); | ||
| 675 | |||
| 676 | MODULE_LICENSE("GPL"); | ||
| 677 | MODULE_DESCRIPTION("VMAC hash algorithm"); | ||
| 678 | |||
