aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorHerbert Xu <herbert@gondor.apana.org.au>2005-07-06 16:52:27 -0400
committerDavid S. Miller <davem@davemloft.net>2005-07-06 16:52:27 -0400
commit6789b2dc455b90efc9c88886c9366adc9abb7347 (patch)
treea962565f31b2c8c0cb112b2ce482cceb3d16caef
parent95477377995aefa2ec1654a9a3777bd57ea99146 (diff)
[PADLOCK] Move fast path work into aes_set_key and upper layer
Most of the work done aes_padlock can be done in aes_set_key. This means that we only have to do it once when the key changes rather than every time we perform an encryption or decryption. This patch also sets cra_alignmask to let the upper layer ensure that the buffers fed to us are aligned correctly. Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au> Signed-off-by: David S. Miller <davem@davemloft.net>
-rw-r--r--drivers/crypto/padlock-aes.c102
-rw-r--r--drivers/crypto/padlock.h22
2 files changed, 52 insertions, 72 deletions
diff --git a/drivers/crypto/padlock-aes.c b/drivers/crypto/padlock-aes.c
index ed708b4427b0..5f28909d4012 100644
--- a/drivers/crypto/padlock-aes.c
+++ b/drivers/crypto/padlock-aes.c
@@ -49,6 +49,7 @@
49#include <linux/errno.h> 49#include <linux/errno.h>
50#include <linux/crypto.h> 50#include <linux/crypto.h>
51#include <linux/interrupt.h> 51#include <linux/interrupt.h>
52#include <linux/kernel.h>
52#include <asm/byteorder.h> 53#include <asm/byteorder.h>
53#include "padlock.h" 54#include "padlock.h"
54 55
@@ -59,8 +60,12 @@
59#define AES_EXTENDED_KEY_SIZE_B (AES_EXTENDED_KEY_SIZE * sizeof(uint32_t)) 60#define AES_EXTENDED_KEY_SIZE_B (AES_EXTENDED_KEY_SIZE * sizeof(uint32_t))
60 61
61struct aes_ctx { 62struct aes_ctx {
62 uint32_t e_data[AES_EXTENDED_KEY_SIZE+4]; 63 uint32_t e_data[AES_EXTENDED_KEY_SIZE];
63 uint32_t d_data[AES_EXTENDED_KEY_SIZE+4]; 64 uint32_t d_data[AES_EXTENDED_KEY_SIZE];
65 struct {
66 struct cword encrypt;
67 struct cword decrypt;
68 } cword;
64 uint32_t *E; 69 uint32_t *E;
65 uint32_t *D; 70 uint32_t *D;
66 int key_length; 71 int key_length;
@@ -280,10 +285,15 @@ aes_hw_extkey_available(uint8_t key_len)
280 return 0; 285 return 0;
281} 286}
282 287
288static inline struct aes_ctx *aes_ctx(void *ctx)
289{
290 return (struct aes_ctx *)ALIGN((unsigned long)ctx, PADLOCK_ALIGNMENT);
291}
292
283static int 293static int
284aes_set_key(void *ctx_arg, const uint8_t *in_key, unsigned int key_len, uint32_t *flags) 294aes_set_key(void *ctx_arg, const uint8_t *in_key, unsigned int key_len, uint32_t *flags)
285{ 295{
286 struct aes_ctx *ctx = ctx_arg; 296 struct aes_ctx *ctx = aes_ctx(ctx_arg);
287 uint32_t i, t, u, v, w; 297 uint32_t i, t, u, v, w;
288 uint32_t P[AES_EXTENDED_KEY_SIZE]; 298 uint32_t P[AES_EXTENDED_KEY_SIZE];
289 uint32_t rounds; 299 uint32_t rounds;
@@ -295,25 +305,36 @@ aes_set_key(void *ctx_arg, const uint8_t *in_key, unsigned int key_len, uint32_t
295 305
296 ctx->key_length = key_len; 306 ctx->key_length = key_len;
297 307
308 /*
309 * If the hardware is capable of generating the extended key
310 * itself we must supply the plain key for both encryption
311 * and decryption.
312 */
298 ctx->E = ctx->e_data; 313 ctx->E = ctx->e_data;
299 ctx->D = ctx->d_data; 314 ctx->D = ctx->e_data;
300
301 /* Ensure 16-Bytes alignmentation of keys for VIA PadLock. */
302 if ((int)(ctx->e_data) & 0x0F)
303 ctx->E += 4 - (((int)(ctx->e_data) & 0x0F) / sizeof (ctx->e_data[0]));
304
305 if ((int)(ctx->d_data) & 0x0F)
306 ctx->D += 4 - (((int)(ctx->d_data) & 0x0F) / sizeof (ctx->d_data[0]));
307 315
308 E_KEY[0] = uint32_t_in (in_key); 316 E_KEY[0] = uint32_t_in (in_key);
309 E_KEY[1] = uint32_t_in (in_key + 4); 317 E_KEY[1] = uint32_t_in (in_key + 4);
310 E_KEY[2] = uint32_t_in (in_key + 8); 318 E_KEY[2] = uint32_t_in (in_key + 8);
311 E_KEY[3] = uint32_t_in (in_key + 12); 319 E_KEY[3] = uint32_t_in (in_key + 12);
312 320
321 /* Prepare control words. */
322 memset(&ctx->cword, 0, sizeof(ctx->cword));
323
324 ctx->cword.decrypt.encdec = 1;
325 ctx->cword.encrypt.rounds = 10 + (key_len - 16) / 4;
326 ctx->cword.decrypt.rounds = ctx->cword.encrypt.rounds;
327 ctx->cword.encrypt.ksize = (key_len - 16) / 8;
328 ctx->cword.decrypt.ksize = ctx->cword.encrypt.ksize;
329
313 /* Don't generate extended keys if the hardware can do it. */ 330 /* Don't generate extended keys if the hardware can do it. */
314 if (aes_hw_extkey_available(key_len)) 331 if (aes_hw_extkey_available(key_len))
315 return 0; 332 return 0;
316 333
334 ctx->D = ctx->d_data;
335 ctx->cword.encrypt.keygen = 1;
336 ctx->cword.decrypt.keygen = 1;
337
317 switch (key_len) { 338 switch (key_len) {
318 case 16: 339 case 16:
319 t = E_KEY[3]; 340 t = E_KEY[3];
@@ -370,9 +391,8 @@ aes_set_key(void *ctx_arg, const uint8_t *in_key, unsigned int key_len, uint32_t
370/* ====== Encryption/decryption routines ====== */ 391/* ====== Encryption/decryption routines ====== */
371 392
372/* This is the real call to PadLock. */ 393/* This is the real call to PadLock. */
373static inline void 394static inline void padlock_xcrypt_ecb(const u8 *input, u8 *output, void *key,
374padlock_xcrypt_ecb(uint8_t *input, uint8_t *output, uint8_t *key, 395 void *control_word, u32 count)
375 void *control_word, uint32_t count)
376{ 396{
377 asm volatile ("pushfl; popfl"); /* enforce key reload. */ 397 asm volatile ("pushfl; popfl"); /* enforce key reload. */
378 asm volatile (".byte 0xf3,0x0f,0xa7,0xc8" /* rep xcryptecb */ 398 asm volatile (".byte 0xf3,0x0f,0xa7,0xc8" /* rep xcryptecb */
@@ -381,66 +401,26 @@ padlock_xcrypt_ecb(uint8_t *input, uint8_t *output, uint8_t *key,
381} 401}
382 402
383static void 403static void
384aes_padlock(void *ctx_arg, uint8_t *out_arg, const uint8_t *in_arg, int encdec)
385{
386 /* Don't blindly modify this structure - the items must
387 fit on 16-Bytes boundaries! */
388 struct padlock_xcrypt_data {
389 uint8_t buf[AES_BLOCK_SIZE];
390 union cword cword;
391 };
392
393 struct aes_ctx *ctx = ctx_arg;
394 char bigbuf[sizeof(struct padlock_xcrypt_data) + 16];
395 struct padlock_xcrypt_data *data;
396 void *key;
397
398 /* Place 'data' at the first 16-Bytes aligned address in 'bigbuf'. */
399 if (((long)bigbuf) & 0x0F)
400 data = (void*)(bigbuf + 16 - ((long)bigbuf & 0x0F));
401 else
402 data = (void*)bigbuf;
403
404 /* Prepare Control word. */
405 memset (data, 0, sizeof(struct padlock_xcrypt_data));
406 data->cword.b.encdec = !encdec; /* in the rest of cryptoapi ENC=1/DEC=0 */
407 data->cword.b.rounds = 10 + (ctx->key_length - 16) / 4;
408 data->cword.b.ksize = (ctx->key_length - 16) / 8;
409
410 /* Is the hardware capable to generate the extended key? */
411 if (!aes_hw_extkey_available(ctx->key_length))
412 data->cword.b.keygen = 1;
413
414 /* ctx->E starts with a plain key - if the hardware is capable
415 to generate the extended key itself we must supply
416 the plain key for both Encryption and Decryption. */
417 if (encdec == CRYPTO_DIR_ENCRYPT || data->cword.b.keygen == 0)
418 key = ctx->E;
419 else
420 key = ctx->D;
421
422 memcpy(data->buf, in_arg, AES_BLOCK_SIZE);
423 padlock_xcrypt_ecb(data->buf, data->buf, key, &data->cword, 1);
424 memcpy(out_arg, data->buf, AES_BLOCK_SIZE);
425}
426
427static void
428aes_encrypt(void *ctx_arg, uint8_t *out, const uint8_t *in) 404aes_encrypt(void *ctx_arg, uint8_t *out, const uint8_t *in)
429{ 405{
430 aes_padlock(ctx_arg, out, in, CRYPTO_DIR_ENCRYPT); 406 struct aes_ctx *ctx = aes_ctx(ctx_arg);
407 padlock_xcrypt_ecb(in, out, ctx->E, &ctx->cword.encrypt, 1);
431} 408}
432 409
433static void 410static void
434aes_decrypt(void *ctx_arg, uint8_t *out, const uint8_t *in) 411aes_decrypt(void *ctx_arg, uint8_t *out, const uint8_t *in)
435{ 412{
436 aes_padlock(ctx_arg, out, in, CRYPTO_DIR_DECRYPT); 413 struct aes_ctx *ctx = aes_ctx(ctx_arg);
414 padlock_xcrypt_ecb(in, out, ctx->D, &ctx->cword.decrypt, 1);
437} 415}
438 416
439static struct crypto_alg aes_alg = { 417static struct crypto_alg aes_alg = {
440 .cra_name = "aes", 418 .cra_name = "aes",
441 .cra_flags = CRYPTO_ALG_TYPE_CIPHER, 419 .cra_flags = CRYPTO_ALG_TYPE_CIPHER,
442 .cra_blocksize = AES_BLOCK_SIZE, 420 .cra_blocksize = AES_BLOCK_SIZE,
443 .cra_ctxsize = sizeof(struct aes_ctx), 421 .cra_ctxsize = sizeof(struct aes_ctx) +
422 PADLOCK_ALIGNMENT,
423 .cra_alignmask = PADLOCK_ALIGNMENT - 1,
444 .cra_module = THIS_MODULE, 424 .cra_module = THIS_MODULE,
445 .cra_list = LIST_HEAD_INIT(aes_alg.cra_list), 425 .cra_list = LIST_HEAD_INIT(aes_alg.cra_list),
446 .cra_u = { 426 .cra_u = {
diff --git a/drivers/crypto/padlock.h b/drivers/crypto/padlock.h
index 7a500605e449..3cf2b7a12348 100644
--- a/drivers/crypto/padlock.h
+++ b/drivers/crypto/padlock.h
@@ -13,18 +13,18 @@
13#ifndef _CRYPTO_PADLOCK_H 13#ifndef _CRYPTO_PADLOCK_H
14#define _CRYPTO_PADLOCK_H 14#define _CRYPTO_PADLOCK_H
15 15
16#define PADLOCK_ALIGNMENT 16
17
16/* Control word. */ 18/* Control word. */
17union cword { 19struct cword {
18 uint32_t cword[4]; 20 int __attribute__ ((__packed__))
19 struct { 21 rounds:4,
20 int rounds:4; 22 algo:3,
21 int algo:3; 23 keygen:1,
22 int keygen:1; 24 interm:1,
23 int interm:1; 25 encdec:1,
24 int encdec:1; 26 ksize:2;
25 int ksize:2; 27} __attribute__ ((__aligned__(PADLOCK_ALIGNMENT)));
26 } b;
27};
28 28
29#define PFX "padlock: " 29#define PFX "padlock: "
30 30