12 files changed, 801 insertions, 80 deletions

diff --git a/crypto/ablkcipher.c b/crypto/ablkcipher.c
index fe980dae1727..98a66103f4f2 100644
--- a/crypto/ablkcipher.c
+++ b/crypto/ablkcipher.c
@@ -24,10 +24,287 @@
 #include <linux/slab.h>
 #include <linux/seq_file.h>
 
+#include <crypto/scatterwalk.h>
+
 #include "internal.h"
 
 static const char *skcipher_default_geniv __read_mostly;
 
+struct ablkcipher_buffer {
+        struct list_head        entry;
+        struct scatter_walk     dst;
+        unsigned int            len;
+        void                    *data;
+};
+
+enum {
+        ABLKCIPHER_WALK_SLOW = 1 << 0,
+};
+
+static inline void ablkcipher_buffer_write(struct ablkcipher_buffer *p)
+{
+        scatterwalk_copychunks(p->data, &p->dst, p->len, 1);
+}
+
+void __ablkcipher_walk_complete(struct ablkcipher_walk *walk)
+{
+        struct ablkcipher_buffer *p, *tmp;
+
+        list_for_each_entry_safe(p, tmp, &walk->buffers, entry) {
+                ablkcipher_buffer_write(p);
+                list_del(&p->entry);
+                kfree(p);
+        }
+}
+EXPORT_SYMBOL_GPL(__ablkcipher_walk_complete);
+
+static inline void ablkcipher_queue_write(struct ablkcipher_walk *walk,
+                                          struct ablkcipher_buffer *p)
+{
+        p->dst = walk->out;
+        list_add_tail(&p->entry, &walk->buffers);
+}
+
+/* Get a spot of the specified length that does not straddle a page.
+ * The caller needs to ensure that there is enough space for this operation.
+ */
+static inline u8 *ablkcipher_get_spot(u8 *start, unsigned int len)
+{
+        u8 *end_page = (u8 *)(((unsigned long)(start + len - 1)) & PAGE_MASK);
+        return max(start, end_page);
+}
+
+static inline unsigned int ablkcipher_done_slow(struct ablkcipher_walk *walk,
+                                                unsigned int bsize)
+{
+        unsigned int n = bsize;
+
+        for (;;) {
+                unsigned int len_this_page = scatterwalk_pagelen(&walk->out);
+
+                if (len_this_page > n)
+                        len_this_page = n;
+                scatterwalk_advance(&walk->out, n);
+                if (n == len_this_page)
+                        break;
+                n -= len_this_page;
+                scatterwalk_start(&walk->out, scatterwalk_sg_next(walk->out.sg));
+        }
+
+        return bsize;
+}
+
+static inline unsigned int ablkcipher_done_fast(struct ablkcipher_walk *walk,
+                                                unsigned int n)
+{
+        scatterwalk_advance(&walk->in, n);
+        scatterwalk_advance(&walk->out, n);
+
+        return n;
+}
+
+static int ablkcipher_walk_next(struct ablkcipher_request *req,
+                                struct ablkcipher_walk *walk);
+
+int ablkcipher_walk_done(struct ablkcipher_request *req,
+                         struct ablkcipher_walk *walk, int err)
+{
+        struct crypto_tfm *tfm = req->base.tfm;
+        unsigned int nbytes = 0;
+
+        if (likely(err >= 0)) {
+                unsigned int n = walk->nbytes - err;
+
+                if (likely(!(walk->flags & ABLKCIPHER_WALK_SLOW)))
+                        n = ablkcipher_done_fast(walk, n);
+                else if (WARN_ON(err)) {
+                        err = -EINVAL;
+                        goto err;
+                } else
+                        n = ablkcipher_done_slow(walk, n);
+
+                nbytes = walk->total - n;
+                err = 0;
+        }
+
+        scatterwalk_done(&walk->in, 0, nbytes);
+        scatterwalk_done(&walk->out, 1, nbytes);
+
+err:
+        walk->total = nbytes;
+        walk->nbytes = nbytes;
+
+        if (nbytes) {
+                crypto_yield(req->base.flags);
+                return ablkcipher_walk_next(req, walk);
+        }
+
+        if (walk->iv != req->info)
+                memcpy(req->info, walk->iv, tfm->crt_ablkcipher.ivsize);
+        if (walk->iv_buffer)
+                kfree(walk->iv_buffer);
+
+        return err;
+}
+EXPORT_SYMBOL_GPL(ablkcipher_walk_done);
+
+static inline int ablkcipher_next_slow(struct ablkcipher_request *req,
+                                       struct ablkcipher_walk *walk,
+                                       unsigned int bsize,
+                                       unsigned int alignmask,
+                                       void **src_p, void **dst_p)
+{
+        unsigned aligned_bsize = ALIGN(bsize, alignmask + 1);
+        struct ablkcipher_buffer *p;
+        void *src, *dst, *base;
+        unsigned int n;
+
+        n = ALIGN(sizeof(struct ablkcipher_buffer), alignmask + 1);
+        n += (aligned_bsize * 3 - (alignmask + 1) +
+              (alignmask & ~(crypto_tfm_ctx_alignment() - 1)));
+
+        p = kmalloc(n, GFP_ATOMIC);
+        if (!p)
+                ablkcipher_walk_done(req, walk, -ENOMEM);
+
+        base = p + 1;
+
+        dst = (u8 *)ALIGN((unsigned long)base, alignmask + 1);
+        src = dst = ablkcipher_get_spot(dst, bsize);
+
+        p->len = bsize;
+        p->data = dst;
+
+        scatterwalk_copychunks(src, &walk->in, bsize, 0);
+
+        ablkcipher_queue_write(walk, p);
+
+        walk->nbytes = bsize;
+        walk->flags |= ABLKCIPHER_WALK_SLOW;
+
+        *src_p = src;
+        *dst_p = dst;
+
+        return 0;
+}
+
+static inline int ablkcipher_copy_iv(struct ablkcipher_walk *walk,
+                                     struct crypto_tfm *tfm,
+                                     unsigned int alignmask)
+{
+        unsigned bs = walk->blocksize;
+        unsigned int ivsize = tfm->crt_ablkcipher.ivsize;
+        unsigned aligned_bs = ALIGN(bs, alignmask + 1);
+        unsigned int size = aligned_bs * 2 + ivsize + max(aligned_bs, ivsize) -
+                            (alignmask + 1);
+        u8 *iv;
+
+        size += alignmask & ~(crypto_tfm_ctx_alignment() - 1);
+        walk->iv_buffer = kmalloc(size, GFP_ATOMIC);
+        if (!walk->iv_buffer)
+                return -ENOMEM;
+
+        iv = (u8 *)ALIGN((unsigned long)walk->iv_buffer, alignmask + 1);
+        iv = ablkcipher_get_spot(iv, bs) + aligned_bs;
+        iv = ablkcipher_get_spot(iv, bs) + aligned_bs;
+        iv = ablkcipher_get_spot(iv, ivsize);
+
+        walk->iv = memcpy(iv, walk->iv, ivsize);
+        return 0;
+}
+
+static inline int ablkcipher_next_fast(struct ablkcipher_request *req,
+                                       struct ablkcipher_walk *walk)
+{
+        walk->src.page = scatterwalk_page(&walk->in);
+        walk->src.offset = offset_in_page(walk->in.offset);
+        walk->dst.page = scatterwalk_page(&walk->out);
+        walk->dst.offset = offset_in_page(walk->out.offset);
+
+        return 0;
+}
+
+static int ablkcipher_walk_next(struct ablkcipher_request *req,
+                                struct ablkcipher_walk *walk)
+{
+        struct crypto_tfm *tfm = req->base.tfm;
+        unsigned int alignmask, bsize, n;
+        void *src, *dst;
+        int err;
+
+        alignmask = crypto_tfm_alg_alignmask(tfm);
+        n = walk->total;
+        if (unlikely(n < crypto_tfm_alg_blocksize(tfm))) {
+                req->base.flags |= CRYPTO_TFM_RES_BAD_BLOCK_LEN;
+                return ablkcipher_walk_done(req, walk, -EINVAL);
+        }
+
+        walk->flags &= ~ABLKCIPHER_WALK_SLOW;
+        src = dst = NULL;
+
+        bsize = min(walk->blocksize, n);
+        n = scatterwalk_clamp(&walk->in, n);
+        n = scatterwalk_clamp(&walk->out, n);
+
+        if (n < bsize ||
+            !scatterwalk_aligned(&walk->in, alignmask) ||
+            !scatterwalk_aligned(&walk->out, alignmask)) {
+                err = ablkcipher_next_slow(req, walk, bsize, alignmask,
+                                           &src, &dst);
+                goto set_phys_lowmem;
+        }
+
+        walk->nbytes = n;
+
+        return ablkcipher_next_fast(req, walk);
+
+set_phys_lowmem:
+        if (err >= 0) {
+                walk->src.page = virt_to_page(src);
+                walk->dst.page = virt_to_page(dst);
+                walk->src.offset = ((unsigned long)src & (PAGE_SIZE - 1));
+                walk->dst.offset = ((unsigned long)dst & (PAGE_SIZE - 1));
+        }
+
+        return err;
+}
+
+static int ablkcipher_walk_first(struct ablkcipher_request *req,
+                                 struct ablkcipher_walk *walk)
+{
+        struct crypto_tfm *tfm = req->base.tfm;
+        unsigned int alignmask;
+
+        alignmask = crypto_tfm_alg_alignmask(tfm);
+        if (WARN_ON_ONCE(in_irq()))
+                return -EDEADLK;
+
+        walk->nbytes = walk->total;
+        if (unlikely(!walk->total))
+                return 0;
+
+        walk->iv_buffer = NULL;
+        walk->iv = req->info;
+        if (unlikely(((unsigned long)walk->iv & alignmask))) {
+                int err = ablkcipher_copy_iv(walk, tfm, alignmask);
+                if (err)
+                        return err;
+        }
+
+        scatterwalk_start(&walk->in, walk->in.sg);
+        scatterwalk_start(&walk->out, walk->out.sg);
+
+        return ablkcipher_walk_next(req, walk);
+}
+
+int ablkcipher_walk_phys(struct ablkcipher_request *req,
+                         struct ablkcipher_walk *walk)
+{
+        walk->blocksize = crypto_tfm_alg_blocksize(req->base.tfm);
+        return ablkcipher_walk_first(req, walk);
+}
+EXPORT_SYMBOL_GPL(ablkcipher_walk_phys);
+
 static int setkey_unaligned(struct crypto_ablkcipher *tfm, const u8 *key,
                             unsigned int keylen)
 {
diff --git a/crypto/algapi.c b/crypto/algapi.c
index 76fae27ed01c..c3cf1a69a47a 100644
--- a/crypto/algapi.c
+++ b/crypto/algapi.c
@@ -544,7 +544,7 @@ int crypto_init_spawn2(struct crypto_spawn *spawn, struct crypto_alg *alg,
 {
         int err = -EINVAL;
 
-        if (frontend && (alg->cra_flags ^ frontend->type) & frontend->maskset)
+        if ((alg->cra_flags ^ frontend->type) & frontend->maskset)
                 goto out;
 
         spawn->frontend = frontend;
diff --git a/crypto/authenc.c b/crypto/authenc.c
index 05eb32e0d949..b9884ee0adb6 100644
--- a/crypto/authenc.c
+++ b/crypto/authenc.c
@@ -181,6 +181,7 @@ static void authenc_verify_ahash_update_done(struct crypto_async_request *areq,
         struct crypto_authenc_ctx *ctx = crypto_aead_ctx(authenc);
         struct authenc_request_ctx *areq_ctx = aead_request_ctx(req);
         struct ahash_request *ahreq = (void *)(areq_ctx->tail + ctx->reqoff);
+        unsigned int cryptlen = req->cryptlen;
 
         if (err)
                 goto out;
@@ -196,6 +197,7 @@ static void authenc_verify_ahash_update_done(struct crypto_async_request *areq,
                 goto out;
 
         authsize = crypto_aead_authsize(authenc);
+        cryptlen -= authsize;
         ihash = ahreq->result + authsize;
         scatterwalk_map_and_copy(ihash, areq_ctx->sg, areq_ctx->cryptlen,
                                  authsize, 0);
@@ -209,7 +211,7 @@ static void authenc_verify_ahash_update_done(struct crypto_async_request *areq,
         ablkcipher_request_set_callback(abreq, aead_request_flags(req),
                                         req->base.complete, req->base.data);
         ablkcipher_request_set_crypt(abreq, req->src, req->dst,
-                                     req->cryptlen, req->iv);
+                                     cryptlen, req->iv);
 
         err = crypto_ablkcipher_decrypt(abreq);
 
@@ -228,11 +230,13 @@ static void authenc_verify_ahash_done(struct crypto_async_request *areq,
         struct crypto_authenc_ctx *ctx = crypto_aead_ctx(authenc);
         struct authenc_request_ctx *areq_ctx = aead_request_ctx(req);
         struct ahash_request *ahreq = (void *)(areq_ctx->tail + ctx->reqoff);
+        unsigned int cryptlen = req->cryptlen;
 
         if (err)
                 goto out;
 
         authsize = crypto_aead_authsize(authenc);
+        cryptlen -= authsize;
         ihash = ahreq->result + authsize;
         scatterwalk_map_and_copy(ihash, areq_ctx->sg, areq_ctx->cryptlen,
                                  authsize, 0);
@@ -246,7 +250,7 @@ static void authenc_verify_ahash_done(struct crypto_async_request *areq,
         ablkcipher_request_set_callback(abreq, aead_request_flags(req),
                                         req->base.complete, req->base.data);
         ablkcipher_request_set_crypt(abreq, req->src, req->dst,
-                                     req->cryptlen, req->iv);
+                                     cryptlen, req->iv);
 
         err = crypto_ablkcipher_decrypt(abreq);
 
diff --git a/crypto/internal.h b/crypto/internal.h
index 2d226362e594..d4384b08ab29 100644
--- a/crypto/internal.h
+++ b/crypto/internal.h
@@ -6,7 +6,7 @@
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by the Free
- * Software Foundation; either version 2 of the License, or (at your option) 
+ * Software Foundation; either version 2 of the License, or (at your option)
  * any later version.
  *
  */
diff --git a/crypto/pcrypt.c b/crypto/pcrypt.c
index 80201241b698..247178cb98ec 100644
--- a/crypto/pcrypt.c
+++ b/crypto/pcrypt.c
@@ -315,16 +315,13 @@ out_free_inst:
         goto out;
 }
 
-static struct crypto_instance *pcrypt_alloc_aead(struct rtattr **tb)
+static struct crypto_instance *pcrypt_alloc_aead(struct rtattr **tb,
+                                                 u32 type, u32 mask)
 {
         struct crypto_instance *inst;
         struct crypto_alg *alg;
-        struct crypto_attr_type *algt;
-
-        algt = crypto_get_attr_type(tb);
 
-        alg = crypto_get_attr_alg(tb, algt->type,
-                                  (algt->mask & CRYPTO_ALG_TYPE_MASK));
+        alg = crypto_get_attr_alg(tb, type, (mask & CRYPTO_ALG_TYPE_MASK));
         if (IS_ERR(alg))
                 return ERR_CAST(alg);
 
@@ -365,7 +362,7 @@ static struct crypto_instance *pcrypt_alloc(struct rtattr **tb)
 
         switch (algt->type & algt->mask & CRYPTO_ALG_TYPE_MASK) {
         case CRYPTO_ALG_TYPE_AEAD:
-                return pcrypt_alloc_aead(tb);
+                return pcrypt_alloc_aead(tb, algt->type, algt->mask);
         }
 
         return ERR_PTR(-EINVAL);
diff --git a/crypto/scatterwalk.c b/crypto/scatterwalk.c
index 3de89a424401..41e529af0773 100644
--- a/crypto/scatterwalk.c
+++ b/crypto/scatterwalk.c
@@ -68,7 +68,7 @@ static void scatterwalk_pagedone(struct scatter_walk *walk, int out,
 
 void scatterwalk_done(struct scatter_walk *walk, int out, int more)
 {
-        if (!offset_in_page(walk->offset) || !more)
+        if (!(scatterwalk_pagelen(walk) & (PAGE_SIZE - 1)) || !more)
                 scatterwalk_pagedone(walk, out, more);
 }
 EXPORT_SYMBOL_GPL(scatterwalk_done);
diff --git a/crypto/shash.c b/crypto/shash.c
index 91f7b9d83881..22fd9433141f 100644
--- a/crypto/shash.c
+++ b/crypto/shash.c
@@ -37,7 +37,7 @@ static int shash_setkey_unaligned(struct crypto_shash *tfm, const u8 *key,
         u8 *buffer, *alignbuffer;
         int err;
 
-        absize = keylen + (alignmask & ~(CRYPTO_MINALIGN - 1));
+        absize = keylen + (alignmask & ~(crypto_tfm_ctx_alignment() - 1));
         buffer = kmalloc(absize, GFP_KERNEL);
         if (!buffer)
                 return -ENOMEM;
diff --git a/crypto/tcrypt.c b/crypto/tcrypt.c
index a35159947a26..3ca68f9fc14d 100644
--- a/crypto/tcrypt.c
+++ b/crypto/tcrypt.c
@@ -394,6 +394,17 @@ out:
         return 0;
 }
 
+static void test_hash_sg_init(struct scatterlist *sg)
+{
+        int i;
+
+        sg_init_table(sg, TVMEMSIZE);
+        for (i = 0; i < TVMEMSIZE; i++) {
+                sg_set_buf(sg + i, tvmem[i], PAGE_SIZE);
+                memset(tvmem[i], 0xff, PAGE_SIZE);
+        }
+}
+
 static void test_hash_speed(const char *algo, unsigned int sec,
                             struct hash_speed *speed)
 {
@@ -423,12 +434,7 @@ static void test_hash_speed(const char *algo, unsigned int sec,
                 goto out;
         }
 
-        sg_init_table(sg, TVMEMSIZE);
-        for (i = 0; i < TVMEMSIZE; i++) {
-                sg_set_buf(sg + i, tvmem[i], PAGE_SIZE);
-                memset(tvmem[i], 0xff, PAGE_SIZE);
-        }
-
+        test_hash_sg_init(sg);
         for (i = 0; speed[i].blen != 0; i++) {
                 if (speed[i].blen > TVMEMSIZE * PAGE_SIZE) {
                         printk(KERN_ERR
@@ -437,6 +443,9 @@ static void test_hash_speed(const char *algo, unsigned int sec,
                         goto out;
                 }
 
+                if (speed[i].klen)
+                        crypto_hash_setkey(tfm, tvmem[0], speed[i].klen);
+
                 printk(KERN_INFO "test%3u "
                        "(%5u byte blocks,%5u bytes per update,%4u updates): ",
                        i, speed[i].blen, speed[i].plen, speed[i].blen / speed[i].plen);
@@ -458,6 +467,250 @@ out:
         crypto_free_hash(tfm);
 }
 
+struct tcrypt_result {
+        struct completion completion;
+        int err;
+};
+
+static void tcrypt_complete(struct crypto_async_request *req, int err)
+{
+        struct tcrypt_result *res = req->data;
+
+        if (err == -EINPROGRESS)
+                return;
+
+        res->err = err;
+        complete(&res->completion);
+}
+
+static inline int do_one_ahash_op(struct ahash_request *req, int ret)
+{
+        if (ret == -EINPROGRESS || ret == -EBUSY) {
+                struct tcrypt_result *tr = req->base.data;
+
+                ret = wait_for_completion_interruptible(&tr->completion);
+                if (!ret)
+                        ret = tr->err;
+                INIT_COMPLETION(tr->completion);
+        }
+        return ret;
+}
+
+static int test_ahash_jiffies_digest(struct ahash_request *req, int blen,
+                                     char *out, int sec)
+{
+        unsigned long start, end;
+        int bcount;
+        int ret;
+
+        for (start = jiffies, end = start + sec * HZ, bcount = 0;
+             time_before(jiffies, end); bcount++) {
+                ret = do_one_ahash_op(req, crypto_ahash_digest(req));
+                if (ret)
+                        return ret;
+        }
+
+        printk("%6u opers/sec, %9lu bytes/sec\n",
+               bcount / sec, ((long)bcount * blen) / sec);
+
+        return 0;
+}
+
+static int test_ahash_jiffies(struct ahash_request *req, int blen,
+                              int plen, char *out, int sec)
+{
+        unsigned long start, end;
+        int bcount, pcount;
+        int ret;
+
+        if (plen == blen)
+                return test_ahash_jiffies_digest(req, blen, out, sec);
+
+        for (start = jiffies, end = start + sec * HZ, bcount = 0;
+             time_before(jiffies, end); bcount++) {
+                ret = crypto_ahash_init(req);
+                if (ret)
+                        return ret;
+                for (pcount = 0; pcount < blen; pcount += plen) {
+                        ret = do_one_ahash_op(req, crypto_ahash_update(req));
+                        if (ret)
+                                return ret;
+                }
+                /* we assume there is enough space in 'out' for the result */
+                ret = do_one_ahash_op(req, crypto_ahash_final(req));
+                if (ret)
+                        return ret;
+        }
+
+        pr_cont("%6u opers/sec, %9lu bytes/sec\n",
+                bcount / sec, ((long)bcount * blen) / sec);
+
+        return 0;
+}
+
+static int test_ahash_cycles_digest(struct ahash_request *req, int blen,
+                                    char *out)
+{
+        unsigned long cycles = 0;
+        int ret, i;
+
+        /* Warm-up run. */
+        for (i = 0; i < 4; i++) {
+                ret = do_one_ahash_op(req, crypto_ahash_digest(req));
+                if (ret)
+                        goto out;
+        }
+
+        /* The real thing. */
+        for (i = 0; i < 8; i++) {
+                cycles_t start, end;
+
+                start = get_cycles();
+
+                ret = do_one_ahash_op(req, crypto_ahash_digest(req));
+                if (ret)
+                        goto out;
+
+                end = get_cycles();
+
+                cycles += end - start;
+        }
+
+out:
+        if (ret)
+                return ret;
+
+        pr_cont("%6lu cycles/operation, %4lu cycles/byte\n",
+                cycles / 8, cycles / (8 * blen));
+
+        return 0;
+}
+
+static int test_ahash_cycles(struct ahash_request *req, int blen,
+                             int plen, char *out)
+{
+        unsigned long cycles = 0;
+        int i, pcount, ret;
+
+        if (plen == blen)
+                return test_ahash_cycles_digest(req, blen, out);
+
+        /* Warm-up run. */
+        for (i = 0; i < 4; i++) {
+                ret = crypto_ahash_init(req);
+                if (ret)
+                        goto out;
+                for (pcount = 0; pcount < blen; pcount += plen) {
+                        ret = do_one_ahash_op(req, crypto_ahash_update(req));
+                        if (ret)
+                                goto out;
+                }
+                ret = do_one_ahash_op(req, crypto_ahash_final(req));
+                if (ret)
+                        goto out;
+        }
+
+        /* The real thing. */
+        for (i = 0; i < 8; i++) {
+                cycles_t start, end;
+
+                start = get_cycles();
+
+                ret = crypto_ahash_init(req);
+                if (ret)
+                        goto out;
+                for (pcount = 0; pcount < blen; pcount += plen) {
+                        ret = do_one_ahash_op(req, crypto_ahash_update(req));
+                        if (ret)
+                                goto out;
+                }
+                ret = do_one_ahash_op(req, crypto_ahash_final(req));
+                if (ret)
+                        goto out;
+
+                end = get_cycles();
+
+                cycles += end - start;
+        }
+
+out:
+        if (ret)
+                return ret;
+
+        pr_cont("%6lu cycles/operation, %4lu cycles/byte\n",
+                cycles / 8, cycles / (8 * blen));
+
+        return 0;
+}
+
+static void test_ahash_speed(const char *algo, unsigned int sec,
+                             struct hash_speed *speed)
+{
+        struct scatterlist sg[TVMEMSIZE];
+        struct tcrypt_result tresult;
+        struct ahash_request *req;
+        struct crypto_ahash *tfm;
+        static char output[1024];
+        int i, ret;
+
+        printk(KERN_INFO "\ntesting speed of async %s\n", algo);
+
+        tfm = crypto_alloc_ahash(algo, 0, 0);
+        if (IS_ERR(tfm)) {
+                pr_err("failed to load transform for %s: %ld\n",
+                       algo, PTR_ERR(tfm));
+                return;
+        }
+
+        if (crypto_ahash_digestsize(tfm) > sizeof(output)) {
+                pr_err("digestsize(%u) > outputbuffer(%zu)\n",
+                       crypto_ahash_digestsize(tfm), sizeof(output));
+                goto out;
+        }
+
+        test_hash_sg_init(sg);
+        req = ahash_request_alloc(tfm, GFP_KERNEL);
+        if (!req) {
+                pr_err("ahash request allocation failure\n");
+                goto out;
+        }
+
+        init_completion(&tresult.completion);
+        ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
+                                   tcrypt_complete, &tresult);
+
+        for (i = 0; speed[i].blen != 0; i++) {
+                if (speed[i].blen > TVMEMSIZE * PAGE_SIZE) {
+                        pr_err("template (%u) too big for tvmem (%lu)\n",
+                               speed[i].blen, TVMEMSIZE * PAGE_SIZE);
+                        break;
+                }
+
+                pr_info("test%3u "
+                        "(%5u byte blocks,%5u bytes per update,%4u updates): ",
+                        i, speed[i].blen, speed[i].plen, speed[i].blen / speed[i].plen);
+
+                ahash_request_set_crypt(req, sg, output, speed[i].plen);
+
+                if (sec)
+                        ret = test_ahash_jiffies(req, speed[i].blen,
+                                                 speed[i].plen, output, sec);
+                else
+                        ret = test_ahash_cycles(req, speed[i].blen,
+                                                speed[i].plen, output);
+
+                if (ret) {
+                        pr_err("hashing failed ret=%d\n", ret);
+                        break;
+                }
+        }
+
+        ahash_request_free(req);
+
+out:
+        crypto_free_ahash(tfm);
+}
+
 static void test_available(void)
 {
         char **name = check;
@@ -881,9 +1134,87 @@ static int do_test(int m)
                 test_hash_speed("rmd320", sec, generic_hash_speed_template);
                 if (mode > 300 && mode < 400) break;
 
+        case 318:
+                test_hash_speed("ghash-generic", sec, hash_speed_template_16);
+                if (mode > 300 && mode < 400) break;
+
         case 399:
                 break;
 
+        case 400:
+                /* fall through */
+
+        case 401:
+                test_ahash_speed("md4", sec, generic_hash_speed_template);
+                if (mode > 400 && mode < 500) break;
+
+        case 402:
+                test_ahash_speed("md5", sec, generic_hash_speed_template);
+                if (mode > 400 && mode < 500) break;
+
+        case 403:
+                test_ahash_speed("sha1", sec, generic_hash_speed_template);
+                if (mode > 400 && mode < 500) break;
+
+        case 404:
+                test_ahash_speed("sha256", sec, generic_hash_speed_template);
+                if (mode > 400 && mode < 500) break;
+
+        case 405:
+                test_ahash_speed("sha384", sec, generic_hash_speed_template);
+                if (mode > 400 && mode < 500) break;
+
+        case 406:
+                test_ahash_speed("sha512", sec, generic_hash_speed_template);
+                if (mode > 400 && mode < 500) break;
+
+        case 407:
+                test_ahash_speed("wp256", sec, generic_hash_speed_template);
+                if (mode > 400 && mode < 500) break;
+
+        case 408:
+                test_ahash_speed("wp384", sec, generic_hash_speed_template);
+                if (mode > 400 && mode < 500) break;
+
+        case 409:
+                test_ahash_speed("wp512", sec, generic_hash_speed_template);
+                if (mode > 400 && mode < 500) break;
+
+        case 410:
+                test_ahash_speed("tgr128", sec, generic_hash_speed_template);
+                if (mode > 400 && mode < 500) break;
+
+        case 411:
+                test_ahash_speed("tgr160", sec, generic_hash_speed_template);
+                if (mode > 400 && mode < 500) break;
+
+        case 412:
+                test_ahash_speed("tgr192", sec, generic_hash_speed_template);
+                if (mode > 400 && mode < 500) break;
+
+        case 413:
+                test_ahash_speed("sha224", sec, generic_hash_speed_template);
+                if (mode > 400 && mode < 500) break;
+
+        case 414:
+                test_ahash_speed("rmd128", sec, generic_hash_speed_template);
+                if (mode > 400 && mode < 500) break;
+
+        case 415:
+                test_ahash_speed("rmd160", sec, generic_hash_speed_template);
+                if (mode > 400 && mode < 500) break;
+
+        case 416:
+                test_ahash_speed("rmd256", sec, generic_hash_speed_template);
+                if (mode > 400 && mode < 500) break;
+
+        case 417:
+                test_ahash_speed("rmd320", sec, generic_hash_speed_template);
+                if (mode > 400 && mode < 500) break;
+
+        case 499:
+                break;
+
         case 1000:
                 test_available();
                 break;
diff --git a/crypto/tcrypt.h b/crypto/tcrypt.h
index 966bbfaf95b1..10cb925132c9 100644
--- a/crypto/tcrypt.h
+++ b/crypto/tcrypt.h
@@ -25,6 +25,7 @@ struct cipher_speed_template {
 struct hash_speed {
         unsigned int blen;      /* buffer length */
         unsigned int plen;      /* per-update length */
+        unsigned int klen;      /* key length */
 };
 
 /*
@@ -83,4 +84,32 @@ static struct hash_speed generic_hash_speed_template[] = {
         {  .blen = 0,   .plen = 0, }
 };
 
+static struct hash_speed hash_speed_template_16[] = {
+        { .blen = 16,   .plen = 16,     .klen = 16, },
+        { .blen = 64,   .plen = 16,     .klen = 16, },
+        { .blen = 64,   .plen = 64,     .klen = 16, },
+        { .blen = 256,  .plen = 16,     .klen = 16, },
+        { .blen = 256,  .plen = 64,     .klen = 16, },
+        { .blen = 256,  .plen = 256,    .klen = 16, },
+        { .blen = 1024, .plen = 16,     .klen = 16, },
+        { .blen = 1024, .plen = 256,    .klen = 16, },
+        { .blen = 1024, .plen = 1024,   .klen = 16, },
+        { .blen = 2048, .plen = 16,     .klen = 16, },
+        { .blen = 2048, .plen = 256,    .klen = 16, },
+        { .blen = 2048, .plen = 1024,   .klen = 16, },
+        { .blen = 2048, .plen = 2048,   .klen = 16, },
+        { .blen = 4096, .plen = 16,     .klen = 16, },
+        { .blen = 4096, .plen = 256,    .klen = 16, },
+        { .blen = 4096, .plen = 1024,   .klen = 16, },
+        { .blen = 4096, .plen = 4096,   .klen = 16, },
+        { .blen = 8192, .plen = 16,     .klen = 16, },
+        { .blen = 8192, .plen = 256,    .klen = 16, },
+        { .blen = 8192, .plen = 1024,   .klen = 16, },
+        { .blen = 8192, .plen = 4096,   .klen = 16, },
+        { .blen = 8192, .plen = 8192,   .klen = 16, },
+
+        /* End marker */
+        {  .blen = 0,   .plen = 0,      .klen = 0, }
+};
+
 #endif  /* _CRYPTO_TCRYPT_H */
diff --git a/crypto/testmgr.c b/crypto/testmgr.c
index c494d7610be1..5c8aaa0cb0b9 100644
--- a/crypto/testmgr.c
+++ b/crypto/testmgr.c
@@ -153,8 +153,21 @@ static void testmgr_free_buf(char *buf[XBUFSIZE])
                 free_page((unsigned long)buf[i]);
 }
 
+static int do_one_async_hash_op(struct ahash_request *req,
+                                struct tcrypt_result *tr,
+                                int ret)
+{
+        if (ret == -EINPROGRESS || ret == -EBUSY) {
+                ret = wait_for_completion_interruptible(&tr->completion);
+                if (!ret)
+                        ret = tr->err;
+                INIT_COMPLETION(tr->completion);
+        }
+        return ret;
+}
+
 static int test_hash(struct crypto_ahash *tfm, struct hash_testvec *template,
-                     unsigned int tcount)
+                     unsigned int tcount, bool use_digest)
 {
         const char *algo = crypto_tfm_alg_driver_name(crypto_ahash_tfm(tfm));
         unsigned int i, j, k, temp;
@@ -206,23 +219,36 @@ static int test_hash(struct crypto_ahash *tfm, struct hash_testvec *template,
                 }
 
                 ahash_request_set_crypt(req, sg, result, template[i].psize);
-                ret = crypto_ahash_digest(req);
-                switch (ret) {
-                case 0:
-                        break;
-                case -EINPROGRESS:
-                case -EBUSY:
-                        ret = wait_for_completion_interruptible(
-                                &tresult.completion);
-                        if (!ret && !(ret = tresult.err)) {
-                                INIT_COMPLETION(tresult.completion);
-                                break;
+                if (use_digest) {
+                        ret = do_one_async_hash_op(req, &tresult,
+                                                   crypto_ahash_digest(req));
+                        if (ret) {
+                                pr_err("alg: hash: digest failed on test %d "
+                                       "for %s: ret=%d\n", j, algo, -ret);
+                                goto out;
+                        }
+                } else {
+                        ret = do_one_async_hash_op(req, &tresult,
+                                                   crypto_ahash_init(req));
+                        if (ret) {
+                                pr_err("alt: hash: init failed on test %d "
+                                       "for %s: ret=%d\n", j, algo, -ret);
+                                goto out;
+                        }
+                        ret = do_one_async_hash_op(req, &tresult,
+                                                   crypto_ahash_update(req));
+                        if (ret) {
+                                pr_err("alt: hash: update failed on test %d "
+                                       "for %s: ret=%d\n", j, algo, -ret);
+                                goto out;
+                        }
+                        ret = do_one_async_hash_op(req, &tresult,
+                                                   crypto_ahash_final(req));
+                        if (ret) {
+                                pr_err("alt: hash: final failed on test %d "
+                                       "for %s: ret=%d\n", j, algo, -ret);
+                                goto out;
                         }
-                        /* fall through */
-                default:
-                        printk(KERN_ERR "alg: hash: digest failed on test %d "
-                               "for %s: ret=%d\n", j, algo, -ret);
-                        goto out;
                 }
 
                 if (memcmp(result, template[i].digest,
@@ -1402,7 +1428,11 @@ static int alg_test_hash(const struct alg_test_desc *desc, const char *driver,
                 return PTR_ERR(tfm);
         }
 
-        err = test_hash(tfm, desc->suite.hash.vecs, desc->suite.hash.count);
+        err = test_hash(tfm, desc->suite.hash.vecs,
+                        desc->suite.hash.count, true);
+        if (!err)
+                err = test_hash(tfm, desc->suite.hash.vecs,
+                                desc->suite.hash.count, false);
 
         crypto_free_ahash(tfm);
         return err;
diff --git a/crypto/testmgr.h b/crypto/testmgr.h
index fb765173d41c..74e35377fd30 100644
--- a/crypto/testmgr.h
+++ b/crypto/testmgr.h
@@ -1669,17 +1669,73 @@ static struct hash_testvec aes_xcbc128_tv_template[] = {
         }
 };
 
-#define VMAC_AES_TEST_VECTORS   1
-static char vmac_string[128] = {'\x01', '\x01', '\x01', '\x01',
+#define VMAC_AES_TEST_VECTORS   8
+static char vmac_string1[128] = {'\x01', '\x01', '\x01', '\x01',
                                 '\x02', '\x03', '\x02', '\x02',
                                 '\x02', '\x04', '\x01', '\x07',
                                 '\x04', '\x01', '\x04', '\x03',};
+static char vmac_string2[128] = {'a', 'b', 'c',};
+static char vmac_string3[128] = {'a', 'b', 'c', 'a', 'b', 'c',
+                                'a', 'b', 'c', 'a', 'b', 'c',
+                                'a', 'b', 'c', 'a', 'b', 'c',
+                                'a', 'b', 'c', 'a', 'b', 'c',
+                                'a', 'b', 'c', 'a', 'b', 'c',
+                                'a', 'b', 'c', 'a', 'b', 'c',
+                                'a', 'b', 'c', 'a', 'b', 'c',
+                                'a', 'b', 'c', 'a', 'b', 'c',
+                                };
+
 static struct hash_testvec aes_vmac128_tv_template[] = {
         {
+                .key    = "\x00\x01\x02\x03\x04\x05\x06\x07"
+                          "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f",
+                .plaintext = NULL,
+                .digest = "\x07\x58\x80\x35\x77\xa4\x7b\x54",
+                .psize  = 0,
+                .ksize  = 16,
+        }, {
+                .key    = "\x00\x01\x02\x03\x04\x05\x06\x07"
+                          "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f",
+                .plaintext = vmac_string1,
+                .digest = "\xce\xf5\x3c\xd3\xae\x68\x8c\xa1",
+                .psize  = 128,
+                .ksize  = 16,
+        }, {
+                .key    = "\x00\x01\x02\x03\x04\x05\x06\x07"
+                          "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f",
+                .plaintext = vmac_string2,
+                .digest = "\xc9\x27\xb0\x73\x81\xbd\x14\x2d",
+                .psize  = 128,
+                .ksize  = 16,
+        }, {
                 .key    = "\x00\x01\x02\x03\x04\x05\x06\x07"
                           "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f",
-                .plaintext = vmac_string,
-                .digest = "\xcb\xd7\x8a\xfd\xb7\x33\x79\xe7",
+                .plaintext = vmac_string3,
+                .digest = "\x8d\x1a\x95\x8c\x98\x47\x0b\x19",
+                .psize  = 128,
+                .ksize  = 16,
+        }, {
+                .key    = "abcdefghijklmnop",
+                .plaintext = NULL,
+                .digest = "\x3b\x89\xa1\x26\x9e\x55\x8f\x84",
+                .psize  = 0,
+                .ksize  = 16,
+        }, {
+                .key    = "abcdefghijklmnop",
+                .plaintext = vmac_string1,
+                .digest = "\xab\x5e\xab\xb0\xf6\x8d\x74\xc2",
+                .psize  = 128,
+                .ksize  = 16,
+        }, {
+                .key    = "abcdefghijklmnop",
+                .plaintext = vmac_string2,
+                .digest = "\x11\x15\x68\x42\x3d\x7b\x09\xdf",
+                .psize  = 128,
+                .ksize  = 16,
+        }, {
+                .key    = "abcdefghijklmnop",
+                .plaintext = vmac_string3,
+                .digest = "\x8b\x32\x8f\xe1\xed\x8f\xfa\xd4",
                 .psize  = 128,
                 .ksize  = 16,
         },
diff --git a/crypto/vmac.c b/crypto/vmac.c
index 0a9468e575de..0999274a27ac 100644
--- a/crypto/vmac.c
+++ b/crypto/vmac.c
@@ -43,6 +43,8 @@ const u64 m63   = UINT64_C(0x7fffffffffffffff);  /* 63-bit mask       */
 const u64 m64   = UINT64_C(0xffffffffffffffff);  /* 64-bit mask       */
 const u64 mpoly = UINT64_C(0x1fffffff1fffffff);  /* Poly key mask     */
 
+#define pe64_to_cpup le64_to_cpup               /* Prefer little endian */
+
 #ifdef __LITTLE_ENDIAN
 #define INDEX_HIGH 1
 #define INDEX_LOW 0
@@ -110,8 +112,8 @@ const u64 mpoly = UINT64_C(0x1fffffff1fffffff);  /* Poly key mask     */
                 int i; u64 th, tl;                                      \
                 rh = rl = 0;                                            \
                 for (i = 0; i < nw; i += 2) {                           \
-                        MUL64(th, tl, le64_to_cpup((mp)+i)+(kp)[i],     \
-                                le64_to_cpup((mp)+i+1)+(kp)[i+1]);      \
+                        MUL64(th, tl, pe64_to_cpup((mp)+i)+(kp)[i],     \
+                                pe64_to_cpup((mp)+i+1)+(kp)[i+1]);      \
                         ADD128(rh, rl, th, tl);                         \
                 }                                                       \
         } while (0)
@@ -121,11 +123,11 @@ const u64 mpoly = UINT64_C(0x1fffffff1fffffff);  /* Poly key mask     */
                 int i; u64 th, tl;                                      \
                 rh1 = rl1 = rh = rl = 0;                                \
                 for (i = 0; i < nw; i += 2) {                           \
-                        MUL64(th, tl, le64_to_cpup((mp)+i)+(kp)[i],     \
-                                le64_to_cpup((mp)+i+1)+(kp)[i+1]);      \
+                        MUL64(th, tl, pe64_to_cpup((mp)+i)+(kp)[i],     \
+                                pe64_to_cpup((mp)+i+1)+(kp)[i+1]);      \
                         ADD128(rh, rl, th, tl);                         \
-                        MUL64(th, tl, le64_to_cpup((mp)+i)+(kp)[i+2],   \
-                                le64_to_cpup((mp)+i+1)+(kp)[i+3]);      \
+                        MUL64(th, tl, pe64_to_cpup((mp)+i)+(kp)[i+2],   \
+                                pe64_to_cpup((mp)+i+1)+(kp)[i+3]);      \
                         ADD128(rh1, rl1, th, tl);                       \
                 }                                                       \
         } while (0)
@@ -136,17 +138,17 @@ const u64 mpoly = UINT64_C(0x1fffffff1fffffff);  /* Poly key mask     */
                 int i; u64 th, tl;                                      \
                 rh = rl = 0;                                            \
                 for (i = 0; i < nw; i += 8) {                           \
-                        MUL64(th, tl, le64_to_cpup((mp)+i)+(kp)[i],     \
-                                le64_to_cpup((mp)+i+1)+(kp)[i+1]);      \
+                        MUL64(th, tl, pe64_to_cpup((mp)+i)+(kp)[i],     \
+                                pe64_to_cpup((mp)+i+1)+(kp)[i+1]);      \
                         ADD128(rh, rl, th, tl);                         \
-                        MUL64(th, tl, le64_to_cpup((mp)+i+2)+(kp)[i+2], \
-                                le64_to_cpup((mp)+i+3)+(kp)[i+3]);      \
+                        MUL64(th, tl, pe64_to_cpup((mp)+i+2)+(kp)[i+2], \
+                                pe64_to_cpup((mp)+i+3)+(kp)[i+3]);      \
                         ADD128(rh, rl, th, tl);                         \
-                        MUL64(th, tl, le64_to_cpup((mp)+i+4)+(kp)[i+4], \
-                                le64_to_cpup((mp)+i+5)+(kp)[i+5]);      \
+                        MUL64(th, tl, pe64_to_cpup((mp)+i+4)+(kp)[i+4], \
+                                pe64_to_cpup((mp)+i+5)+(kp)[i+5]);      \
                         ADD128(rh, rl, th, tl);                         \
-                        MUL64(th, tl, le64_to_cpup((mp)+i+6)+(kp)[i+6], \
-                                le64_to_cpup((mp)+i+7)+(kp)[i+7]);      \
+                        MUL64(th, tl, pe64_to_cpup((mp)+i+6)+(kp)[i+6], \
+                                pe64_to_cpup((mp)+i+7)+(kp)[i+7]);      \
                         ADD128(rh, rl, th, tl);                         \
                 }                                                       \
         } while (0)
@@ -156,29 +158,29 @@ const u64 mpoly = UINT64_C(0x1fffffff1fffffff);  /* Poly key mask     */
                 int i; u64 th, tl;                                      \
                 rh1 = rl1 = rh = rl = 0;                                \
                 for (i = 0; i < nw; i += 8) {                           \
-                        MUL64(th, tl, le64_to_cpup((mp)+i)+(kp)[i],     \
-                                le64_to_cpup((mp)+i+1)+(kp)[i+1]);      \
+                        MUL64(th, tl, pe64_to_cpup((mp)+i)+(kp)[i],     \
+                                pe64_to_cpup((mp)+i+1)+(kp)[i+1]);      \
                         ADD128(rh, rl, th, tl);                         \
-                        MUL64(th, tl, le64_to_cpup((mp)+i)+(kp)[i+2],   \
-                                le64_to_cpup((mp)+i+1)+(kp)[i+3]);      \
+                        MUL64(th, tl, pe64_to_cpup((mp)+i)+(kp)[i+2],   \
+                                pe64_to_cpup((mp)+i+1)+(kp)[i+3]);      \
                         ADD128(rh1, rl1, th, tl);                       \
-                        MUL64(th, tl, le64_to_cpup((mp)+i+2)+(kp)[i+2], \
-                                le64_to_cpup((mp)+i+3)+(kp)[i+3]);      \
+                        MUL64(th, tl, pe64_to_cpup((mp)+i+2)+(kp)[i+2], \
+                                pe64_to_cpup((mp)+i+3)+(kp)[i+3]);      \
                         ADD128(rh, rl, th, tl);                         \
-                        MUL64(th, tl, le64_to_cpup((mp)+i+2)+(kp)[i+4], \
-                                le64_to_cpup((mp)+i+3)+(kp)[i+5]);      \
+                        MUL64(th, tl, pe64_to_cpup((mp)+i+2)+(kp)[i+4], \
+                                pe64_to_cpup((mp)+i+3)+(kp)[i+5]);      \
                         ADD128(rh1, rl1, th, tl);                       \
-                        MUL64(th, tl, le64_to_cpup((mp)+i+4)+(kp)[i+4], \
-                                le64_to_cpup((mp)+i+5)+(kp)[i+5]);      \
+                        MUL64(th, tl, pe64_to_cpup((mp)+i+4)+(kp)[i+4], \
+                                pe64_to_cpup((mp)+i+5)+(kp)[i+5]);      \
                         ADD128(rh, rl, th, tl);                         \
-                        MUL64(th, tl, le64_to_cpup((mp)+i+4)+(kp)[i+6], \
-                                le64_to_cpup((mp)+i+5)+(kp)[i+7]);      \
+                        MUL64(th, tl, pe64_to_cpup((mp)+i+4)+(kp)[i+6], \
+                                pe64_to_cpup((mp)+i+5)+(kp)[i+7]);      \
                         ADD128(rh1, rl1, th, tl);                       \
-                        MUL64(th, tl, le64_to_cpup((mp)+i+6)+(kp)[i+6], \
-                                le64_to_cpup((mp)+i+7)+(kp)[i+7]);      \
+                        MUL64(th, tl, pe64_to_cpup((mp)+i+6)+(kp)[i+6], \
+                                pe64_to_cpup((mp)+i+7)+(kp)[i+7]);      \
                         ADD128(rh, rl, th, tl);                         \
-                        MUL64(th, tl, le64_to_cpup((mp)+i+6)+(kp)[i+8], \
-                                le64_to_cpup((mp)+i+7)+(kp)[i+9]);      \
+                        MUL64(th, tl, pe64_to_cpup((mp)+i+6)+(kp)[i+8], \
+                                pe64_to_cpup((mp)+i+7)+(kp)[i+9]);      \
                         ADD128(rh1, rl1, th, tl);                       \
                 }                                                       \
         } while (0)
@@ -216,8 +218,8 @@ const u64 mpoly = UINT64_C(0x1fffffff1fffffff);  /* Poly key mask     */
                 int i;                                                  \
                 rh = rl = t = 0;                                        \
                 for (i = 0; i < nw; i += 2)  {                          \
-                        t1 = le64_to_cpup(mp+i) + kp[i];                \
-                        t2 = le64_to_cpup(mp+i+1) + kp[i+1];            \
+                        t1 = pe64_to_cpup(mp+i) + kp[i];                \
+                        t2 = pe64_to_cpup(mp+i+1) + kp[i+1];            \
                         m2 = MUL32(t1 >> 32, t2);                       \
                         m1 = MUL32(t1, t2 >> 32);                       \
                         ADD128(rh, rl, MUL32(t1 >> 32, t2 >> 32),       \
@@ -322,8 +324,7 @@ static void vhash_abort(struct vmac_ctx *ctx)
         ctx->first_block_processed = 0;
 }
 
-static u64 l3hash(u64 p1, u64 p2,
-                        u64 k1, u64 k2, u64 len)
+static u64 l3hash(u64 p1, u64 p2, u64 k1, u64 k2, u64 len)
 {
         u64 rh, rl, t, z = 0;
 
@@ -474,7 +475,7 @@ static u64 vmac(unsigned char m[], unsigned int mbytes,
         }
         p = be64_to_cpup(out_p + i);
         h = vhash(m, mbytes, (u64 *)0, &ctx->__vmac_ctx);
-        return p + h;
+        return le64_to_cpu(p + h);
 }
 
 static int vmac_set_key(unsigned char user_key[], struct vmac_ctx_t *ctx)
@@ -549,10 +550,6 @@ static int vmac_setkey(struct crypto_shash *parent,
 
 static int vmac_init(struct shash_desc *pdesc)
 {
-        struct crypto_shash *parent = pdesc->tfm;
-        struct vmac_ctx_t *ctx = crypto_shash_ctx(parent);
-
-        memset(&ctx->__vmac_ctx, 0, sizeof(struct vmac_ctx));
         return 0;
 }