aboutsummaryrefslogtreecommitdiffstats
path: root/drivers
diff options
context:
space:
mode:
Diffstat (limited to 'drivers')
-rw-r--r--drivers/char/hw_random/Kconfig13
-rw-r--r--drivers/char/hw_random/Makefile1
-rw-r--r--drivers/char/hw_random/bcm63xx-rng.c120
-rw-r--r--drivers/char/hw_random/core.c45
-rw-r--r--drivers/char/hw_random/exynos-rng.c12
-rw-r--r--drivers/char/hw_random/iproc-rng200.c239
-rw-r--r--drivers/char/hw_random/msm-rng.c11
-rw-r--r--drivers/char/hw_random/octeon-rng.c4
-rw-r--r--drivers/char/hw_random/omap-rng.c23
-rw-r--r--drivers/char/hw_random/pseries-rng.c4
-rw-r--r--drivers/char/hw_random/xgene-rng.c10
-rw-r--r--drivers/crypto/Kconfig24
-rw-r--r--drivers/crypto/Makefile2
-rw-r--r--drivers/crypto/amcc/crypto4xx_core.c6
-rw-r--r--drivers/crypto/atmel-aes.c26
-rw-r--r--drivers/crypto/atmel-sha.c37
-rw-r--r--drivers/crypto/atmel-tdes.c3
-rw-r--r--drivers/crypto/caam/caamhash.c1
-rw-r--r--drivers/crypto/caam/caamrng.c6
-rw-r--r--drivers/crypto/ccp/Makefile9
-rw-r--r--drivers/crypto/ccp/ccp-crypto-aes-cmac.c12
-rw-r--r--drivers/crypto/ccp/ccp-crypto-aes-xts.c4
-rw-r--r--drivers/crypto/ccp/ccp-crypto-aes.c3
-rw-r--r--drivers/crypto/ccp/ccp-crypto-main.c5
-rw-r--r--drivers/crypto/ccp/ccp-crypto-sha.c12
-rw-r--r--drivers/crypto/ccp/ccp-crypto.h3
-rw-r--r--drivers/crypto/ccp/ccp-dev.c7
-rw-r--r--drivers/crypto/ccp/ccp-dev.h12
-rw-r--r--drivers/crypto/ccp/ccp-ops.c24
-rw-r--r--drivers/crypto/ccp/ccp-pci.c21
-rw-r--r--drivers/crypto/ccp/ccp-platform.c111
-rw-r--r--drivers/crypto/img-hash.c1029
-rw-r--r--drivers/crypto/mxs-dcp.c2
-rw-r--r--drivers/crypto/omap-aes.c14
-rw-r--r--drivers/crypto/omap-sham.c2
-rw-r--r--drivers/crypto/qat/qat_common/adf_accel_devices.h1
-rw-r--r--drivers/crypto/qat/qat_common/adf_accel_engine.c35
-rw-r--r--drivers/crypto/qat/qat_common/adf_aer.c21
-rw-r--r--drivers/crypto/qat/qat_common/adf_cfg.c5
-rw-r--r--drivers/crypto/qat/qat_common/adf_cfg_strings.h10
-rw-r--r--drivers/crypto/qat/qat_common/adf_common_drv.h2
-rw-r--r--drivers/crypto/qat/qat_common/adf_ctl_drv.c66
-rw-r--r--drivers/crypto/qat/qat_common/adf_dev_mgr.c3
-rw-r--r--drivers/crypto/qat/qat_common/adf_init.c88
-rw-r--r--drivers/crypto/qat/qat_common/adf_transport.c31
-rw-r--r--drivers/crypto/qat/qat_common/adf_transport_debug.c2
-rw-r--r--drivers/crypto/qat/qat_common/icp_qat_hw.h2
-rw-r--r--drivers/crypto/qat/qat_common/qat_crypto.c9
-rw-r--r--drivers/crypto/qat/qat_common/qat_hal.c6
-rw-r--r--drivers/crypto/qat/qat_dh895xcc/adf_admin.c3
-rw-r--r--drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c3
-rw-r--r--drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.h6
-rw-r--r--drivers/crypto/qat/qat_dh895xcc/adf_drv.c4
-rw-r--r--drivers/crypto/qat/qat_dh895xcc/adf_isr.c15
-rw-r--r--drivers/crypto/sahara.c51
-rw-r--r--drivers/crypto/talitos.c17
-rw-r--r--drivers/crypto/ux500/hash/hash_core.c2
-rw-r--r--drivers/crypto/vmx/Kconfig8
-rw-r--r--drivers/crypto/vmx/Makefile19
-rw-r--r--drivers/crypto/vmx/aes.c139
-rw-r--r--drivers/crypto/vmx/aes_cbc.c184
-rw-r--r--drivers/crypto/vmx/aes_ctr.c167
-rw-r--r--drivers/crypto/vmx/aesp8-ppc.h20
-rw-r--r--drivers/crypto/vmx/aesp8-ppc.pl1930
-rw-r--r--drivers/crypto/vmx/ghash.c214
-rw-r--r--drivers/crypto/vmx/ghashp8-ppc.pl228
-rw-r--r--drivers/crypto/vmx/ppc-xlate.pl207
-rw-r--r--drivers/crypto/vmx/vmx.c88
68 files changed, 5053 insertions, 390 deletions
diff --git a/drivers/char/hw_random/Kconfig b/drivers/char/hw_random/Kconfig
index de57b38809c7..f48cf11c655e 100644
--- a/drivers/char/hw_random/Kconfig
+++ b/drivers/char/hw_random/Kconfig
@@ -101,6 +101,19 @@ config HW_RANDOM_BCM2835
101 101
102 If unsure, say Y. 102 If unsure, say Y.
103 103
104config HW_RANDOM_IPROC_RNG200
105 tristate "Broadcom iProc RNG200 support"
106 depends on ARCH_BCM_IPROC
107 default HW_RANDOM
108 ---help---
109 This driver provides kernel-side support for the RNG200
110 hardware found on the Broadcom iProc SoCs.
111
112 To compile this driver as a module, choose M here: the
113 module will be called iproc-rng200
114
115 If unsure, say Y.
116
104config HW_RANDOM_GEODE 117config HW_RANDOM_GEODE
105 tristate "AMD Geode HW Random Number Generator support" 118 tristate "AMD Geode HW Random Number Generator support"
106 depends on X86_32 && PCI 119 depends on X86_32 && PCI
diff --git a/drivers/char/hw_random/Makefile b/drivers/char/hw_random/Makefile
index 0b4cd57f4e24..055bb01510ad 100644
--- a/drivers/char/hw_random/Makefile
+++ b/drivers/char/hw_random/Makefile
@@ -28,5 +28,6 @@ obj-$(CONFIG_HW_RANDOM_POWERNV) += powernv-rng.o
28obj-$(CONFIG_HW_RANDOM_EXYNOS) += exynos-rng.o 28obj-$(CONFIG_HW_RANDOM_EXYNOS) += exynos-rng.o
29obj-$(CONFIG_HW_RANDOM_TPM) += tpm-rng.o 29obj-$(CONFIG_HW_RANDOM_TPM) += tpm-rng.o
30obj-$(CONFIG_HW_RANDOM_BCM2835) += bcm2835-rng.o 30obj-$(CONFIG_HW_RANDOM_BCM2835) += bcm2835-rng.o
31obj-$(CONFIG_HW_RANDOM_IPROC_RNG200) += iproc-rng200.o
31obj-$(CONFIG_HW_RANDOM_MSM) += msm-rng.o 32obj-$(CONFIG_HW_RANDOM_MSM) += msm-rng.o
32obj-$(CONFIG_HW_RANDOM_XGENE) += xgene-rng.o 33obj-$(CONFIG_HW_RANDOM_XGENE) += xgene-rng.o
diff --git a/drivers/char/hw_random/bcm63xx-rng.c b/drivers/char/hw_random/bcm63xx-rng.c
index ba6a65ac023b..d1494ecd9e11 100644
--- a/drivers/char/hw_random/bcm63xx-rng.c
+++ b/drivers/char/hw_random/bcm63xx-rng.c
@@ -13,24 +13,37 @@
13#include <linux/platform_device.h> 13#include <linux/platform_device.h>
14#include <linux/hw_random.h> 14#include <linux/hw_random.h>
15 15
16#include <bcm63xx_io.h> 16#define RNG_CTRL 0x00
17#include <bcm63xx_regs.h> 17#define RNG_EN (1 << 0)
18
19#define RNG_STAT 0x04
20#define RNG_AVAIL_MASK (0xff000000)
21
22#define RNG_DATA 0x08
23#define RNG_THRES 0x0c
24#define RNG_MASK 0x10
18 25
19struct bcm63xx_rng_priv { 26struct bcm63xx_rng_priv {
27 struct hwrng rng;
20 struct clk *clk; 28 struct clk *clk;
21 void __iomem *regs; 29 void __iomem *regs;
22}; 30};
23 31
24#define to_rng_priv(rng) ((struct bcm63xx_rng_priv *)rng->priv) 32#define to_rng_priv(rng) container_of(rng, struct bcm63xx_rng_priv, rng)
25 33
26static int bcm63xx_rng_init(struct hwrng *rng) 34static int bcm63xx_rng_init(struct hwrng *rng)
27{ 35{
28 struct bcm63xx_rng_priv *priv = to_rng_priv(rng); 36 struct bcm63xx_rng_priv *priv = to_rng_priv(rng);
29 u32 val; 37 u32 val;
38 int error;
39
40 error = clk_prepare_enable(priv->clk);
41 if (error)
42 return error;
30 43
31 val = bcm_readl(priv->regs + RNG_CTRL); 44 val = __raw_readl(priv->regs + RNG_CTRL);
32 val |= RNG_EN; 45 val |= RNG_EN;
33 bcm_writel(val, priv->regs + RNG_CTRL); 46 __raw_writel(val, priv->regs + RNG_CTRL);
34 47
35 return 0; 48 return 0;
36} 49}
@@ -40,23 +53,25 @@ static void bcm63xx_rng_cleanup(struct hwrng *rng)
40 struct bcm63xx_rng_priv *priv = to_rng_priv(rng); 53 struct bcm63xx_rng_priv *priv = to_rng_priv(rng);
41 u32 val; 54 u32 val;
42 55
43 val = bcm_readl(priv->regs + RNG_CTRL); 56 val = __raw_readl(priv->regs + RNG_CTRL);
44 val &= ~RNG_EN; 57 val &= ~RNG_EN;
45 bcm_writel(val, priv->regs + RNG_CTRL); 58 __raw_writel(val, priv->regs + RNG_CTRL);
59
60 clk_didsable_unprepare(prov->clk);
46} 61}
47 62
48static int bcm63xx_rng_data_present(struct hwrng *rng, int wait) 63static int bcm63xx_rng_data_present(struct hwrng *rng, int wait)
49{ 64{
50 struct bcm63xx_rng_priv *priv = to_rng_priv(rng); 65 struct bcm63xx_rng_priv *priv = to_rng_priv(rng);
51 66
52 return bcm_readl(priv->regs + RNG_STAT) & RNG_AVAIL_MASK; 67 return __raw_readl(priv->regs + RNG_STAT) & RNG_AVAIL_MASK;
53} 68}
54 69
55static int bcm63xx_rng_data_read(struct hwrng *rng, u32 *data) 70static int bcm63xx_rng_data_read(struct hwrng *rng, u32 *data)
56{ 71{
57 struct bcm63xx_rng_priv *priv = to_rng_priv(rng); 72 struct bcm63xx_rng_priv *priv = to_rng_priv(rng);
58 73
59 *data = bcm_readl(priv->regs + RNG_DATA); 74 *data = __raw_readl(priv->regs + RNG_DATA);
60 75
61 return 4; 76 return 4;
62} 77}
@@ -72,94 +87,53 @@ static int bcm63xx_rng_probe(struct platform_device *pdev)
72 r = platform_get_resource(pdev, IORESOURCE_MEM, 0); 87 r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
73 if (!r) { 88 if (!r) {
74 dev_err(&pdev->dev, "no iomem resource\n"); 89 dev_err(&pdev->dev, "no iomem resource\n");
75 ret = -ENXIO; 90 return -ENXIO;
76 goto out;
77 } 91 }
78 92
79 priv = kzalloc(sizeof(*priv), GFP_KERNEL); 93 priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL);
80 if (!priv) { 94 if (!priv)
81 dev_err(&pdev->dev, "no memory for private structure\n"); 95 return -ENOMEM;
82 ret = -ENOMEM; 96
83 goto out; 97 priv->rng.name = pdev->name;
98 priv->rng.init = bcm63xx_rng_init;
99 priv->rng.cleanup = bcm63xx_rng_cleanup;
100 prov->rng.data_present = bcm63xx_rng_data_present;
101 priv->rng.data_read = bcm63xx_rng_data_read;
102
103 priv->clk = devm_clk_get(&pdev->dev, "ipsec");
104 if (IS_ERR(priv->clk)) {
105 error = PTR_ERR(priv->clk);
106 dev_err(&pdev->dev, "no clock for device: %d\n", error);
107 return error;
84 } 108 }
85 109
86 rng = kzalloc(sizeof(*rng), GFP_KERNEL);
87 if (!rng) {
88 dev_err(&pdev->dev, "no memory for rng structure\n");
89 ret = -ENOMEM;
90 goto out_free_priv;
91 }
92
93 platform_set_drvdata(pdev, rng);
94 rng->priv = (unsigned long)priv;
95 rng->name = pdev->name;
96 rng->init = bcm63xx_rng_init;
97 rng->cleanup = bcm63xx_rng_cleanup;
98 rng->data_present = bcm63xx_rng_data_present;
99 rng->data_read = bcm63xx_rng_data_read;
100
101 clk = clk_get(&pdev->dev, "ipsec");
102 if (IS_ERR(clk)) {
103 dev_err(&pdev->dev, "no clock for device\n");
104 ret = PTR_ERR(clk);
105 goto out_free_rng;
106 }
107
108 priv->clk = clk;
109
110 if (!devm_request_mem_region(&pdev->dev, r->start, 110 if (!devm_request_mem_region(&pdev->dev, r->start,
111 resource_size(r), pdev->name)) { 111 resource_size(r), pdev->name)) {
112 dev_err(&pdev->dev, "request mem failed"); 112 dev_err(&pdev->dev, "request mem failed");
113 ret = -ENOMEM; 113 return -EBUSY;
114 goto out_free_rng;
115 } 114 }
116 115
117 priv->regs = devm_ioremap_nocache(&pdev->dev, r->start, 116 priv->regs = devm_ioremap_nocache(&pdev->dev, r->start,
118 resource_size(r)); 117 resource_size(r));
119 if (!priv->regs) { 118 if (!priv->regs) {
120 dev_err(&pdev->dev, "ioremap failed"); 119 dev_err(&pdev->dev, "ioremap failed");
121 ret = -ENOMEM; 120 return -ENOMEM;
122 goto out_free_rng;
123 } 121 }
124 122
125 clk_enable(clk); 123 error = devm_hwrng_register(&pdev->dev, &priv->rng);
126 124 if (error) {
127 ret = hwrng_register(rng); 125 dev_err(&pdev->dev, "failed to register rng device: %d\n",
128 if (ret) { 126 error);
129 dev_err(&pdev->dev, "failed to register rng device\n"); 127 return error;
130 goto out_clk_disable;
131 } 128 }
132 129
133 dev_info(&pdev->dev, "registered RNG driver\n"); 130 dev_info(&pdev->dev, "registered RNG driver\n");
134 131
135 return 0; 132 return 0;
136
137out_clk_disable:
138 clk_disable(clk);
139out_free_rng:
140 kfree(rng);
141out_free_priv:
142 kfree(priv);
143out:
144 return ret;
145}
146
147static int bcm63xx_rng_remove(struct platform_device *pdev)
148{
149 struct hwrng *rng = platform_get_drvdata(pdev);
150 struct bcm63xx_rng_priv *priv = to_rng_priv(rng);
151
152 hwrng_unregister(rng);
153 clk_disable(priv->clk);
154 kfree(priv);
155 kfree(rng);
156
157 return 0;
158} 133}
159 134
160static struct platform_driver bcm63xx_rng_driver = { 135static struct platform_driver bcm63xx_rng_driver = {
161 .probe = bcm63xx_rng_probe, 136 .probe = bcm63xx_rng_probe,
162 .remove = bcm63xx_rng_remove,
163 .driver = { 137 .driver = {
164 .name = "bcm63xx-rng", 138 .name = "bcm63xx-rng",
165 }, 139 },
diff --git a/drivers/char/hw_random/core.c b/drivers/char/hw_random/core.c
index 32a8a867f7f8..571ef61f8ea9 100644
--- a/drivers/char/hw_random/core.c
+++ b/drivers/char/hw_random/core.c
@@ -179,7 +179,8 @@ skip_init:
179 add_early_randomness(rng); 179 add_early_randomness(rng);
180 180
181 current_quality = rng->quality ? : default_quality; 181 current_quality = rng->quality ? : default_quality;
182 current_quality &= 1023; 182 if (current_quality > 1024)
183 current_quality = 1024;
183 184
184 if (current_quality == 0 && hwrng_fill) 185 if (current_quality == 0 && hwrng_fill)
185 kthread_stop(hwrng_fill); 186 kthread_stop(hwrng_fill);
@@ -536,6 +537,48 @@ void hwrng_unregister(struct hwrng *rng)
536} 537}
537EXPORT_SYMBOL_GPL(hwrng_unregister); 538EXPORT_SYMBOL_GPL(hwrng_unregister);
538 539
540static void devm_hwrng_release(struct device *dev, void *res)
541{
542 hwrng_unregister(*(struct hwrng **)res);
543}
544
545static int devm_hwrng_match(struct device *dev, void *res, void *data)
546{
547 struct hwrng **r = res;
548
549 if (WARN_ON(!r || !*r))
550 return 0;
551
552 return *r == data;
553}
554
555int devm_hwrng_register(struct device *dev, struct hwrng *rng)
556{
557 struct hwrng **ptr;
558 int error;
559
560 ptr = devres_alloc(devm_hwrng_release, sizeof(*ptr), GFP_KERNEL);
561 if (!ptr)
562 return -ENOMEM;
563
564 error = hwrng_register(rng);
565 if (error) {
566 devres_free(ptr);
567 return error;
568 }
569
570 *ptr = rng;
571 devres_add(dev, ptr);
572 return 0;
573}
574EXPORT_SYMBOL_GPL(devm_hwrng_register);
575
576void devm_hwrng_unregister(struct device *dev, struct hwrng *rng)
577{
578 devres_release(dev, devm_hwrng_release, devm_hwrng_match, rng);
579}
580EXPORT_SYMBOL_GPL(devm_hwrng_unregister);
581
539static int __init hwrng_modinit(void) 582static int __init hwrng_modinit(void)
540{ 583{
541 return register_miscdev(); 584 return register_miscdev();
diff --git a/drivers/char/hw_random/exynos-rng.c b/drivers/char/hw_random/exynos-rng.c
index fed0830bf724..dc4701fd814f 100644
--- a/drivers/char/hw_random/exynos-rng.c
+++ b/drivers/char/hw_random/exynos-rng.c
@@ -131,16 +131,7 @@ static int exynos_rng_probe(struct platform_device *pdev)
131 pm_runtime_use_autosuspend(&pdev->dev); 131 pm_runtime_use_autosuspend(&pdev->dev);
132 pm_runtime_enable(&pdev->dev); 132 pm_runtime_enable(&pdev->dev);
133 133
134 return hwrng_register(&exynos_rng->rng); 134 return devm_hwrng_register(&pdev->dev, &exynos_rng->rng);
135}
136
137static int exynos_rng_remove(struct platform_device *pdev)
138{
139 struct exynos_rng *exynos_rng = platform_get_drvdata(pdev);
140
141 hwrng_unregister(&exynos_rng->rng);
142
143 return 0;
144} 135}
145 136
146#ifdef CONFIG_PM 137#ifdef CONFIG_PM
@@ -172,7 +163,6 @@ static struct platform_driver exynos_rng_driver = {
172 .pm = &exynos_rng_pm_ops, 163 .pm = &exynos_rng_pm_ops,
173 }, 164 },
174 .probe = exynos_rng_probe, 165 .probe = exynos_rng_probe,
175 .remove = exynos_rng_remove,
176}; 166};
177 167
178module_platform_driver(exynos_rng_driver); 168module_platform_driver(exynos_rng_driver);
diff --git a/drivers/char/hw_random/iproc-rng200.c b/drivers/char/hw_random/iproc-rng200.c
new file mode 100644
index 000000000000..3eaf7cb96d36
--- /dev/null
+++ b/drivers/char/hw_random/iproc-rng200.c
@@ -0,0 +1,239 @@
1/*
2* Copyright (C) 2015 Broadcom Corporation
3*
4* This program is free software; you can redistribute it and/or
5* modify it under the terms of the GNU General Public License as
6* published by the Free Software Foundation version 2.
7*
8* This program is distributed "as is" WITHOUT ANY WARRANTY of any
9* kind, whether express or implied; without even the implied warranty
10* of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11* GNU General Public License for more details.
12*/
13/*
14 * DESCRIPTION: The Broadcom iProc RNG200 Driver
15 */
16
17#include <linux/hw_random.h>
18#include <linux/init.h>
19#include <linux/io.h>
20#include <linux/kernel.h>
21#include <linux/module.h>
22#include <linux/of_address.h>
23#include <linux/of_platform.h>
24#include <linux/platform_device.h>
25#include <linux/delay.h>
26
27/* Registers */
28#define RNG_CTRL_OFFSET 0x00
29#define RNG_CTRL_RNG_RBGEN_MASK 0x00001FFF
30#define RNG_CTRL_RNG_RBGEN_ENABLE 0x00000001
31#define RNG_CTRL_RNG_RBGEN_DISABLE 0x00000000
32
33#define RNG_SOFT_RESET_OFFSET 0x04
34#define RNG_SOFT_RESET 0x00000001
35
36#define RBG_SOFT_RESET_OFFSET 0x08
37#define RBG_SOFT_RESET 0x00000001
38
39#define RNG_INT_STATUS_OFFSET 0x18
40#define RNG_INT_STATUS_MASTER_FAIL_LOCKOUT_IRQ_MASK 0x80000000
41#define RNG_INT_STATUS_STARTUP_TRANSITIONS_MET_IRQ_MASK 0x00020000
42#define RNG_INT_STATUS_NIST_FAIL_IRQ_MASK 0x00000020
43#define RNG_INT_STATUS_TOTAL_BITS_COUNT_IRQ_MASK 0x00000001
44
45#define RNG_FIFO_DATA_OFFSET 0x20
46
47#define RNG_FIFO_COUNT_OFFSET 0x24
48#define RNG_FIFO_COUNT_RNG_FIFO_COUNT_MASK 0x000000FF
49
50struct iproc_rng200_dev {
51 struct hwrng rng;
52 void __iomem *base;
53};
54
55#define to_rng_priv(rng) container_of(rng, struct iproc_rng200_dev, rng)
56
57static void iproc_rng200_restart(void __iomem *rng_base)
58{
59 uint32_t val;
60
61 /* Disable RBG */
62 val = ioread32(rng_base + RNG_CTRL_OFFSET);
63 val &= ~RNG_CTRL_RNG_RBGEN_MASK;
64 val |= RNG_CTRL_RNG_RBGEN_DISABLE;
65 iowrite32(val, rng_base + RNG_CTRL_OFFSET);
66
67 /* Clear all interrupt status */
68 iowrite32(0xFFFFFFFFUL, rng_base + RNG_INT_STATUS_OFFSET);
69
70 /* Reset RNG and RBG */
71 val = ioread32(rng_base + RBG_SOFT_RESET_OFFSET);
72 val |= RBG_SOFT_RESET;
73 iowrite32(val, rng_base + RBG_SOFT_RESET_OFFSET);
74
75 val = ioread32(rng_base + RNG_SOFT_RESET_OFFSET);
76 val |= RNG_SOFT_RESET;
77 iowrite32(val, rng_base + RNG_SOFT_RESET_OFFSET);
78
79 val = ioread32(rng_base + RNG_SOFT_RESET_OFFSET);
80 val &= ~RNG_SOFT_RESET;
81 iowrite32(val, rng_base + RNG_SOFT_RESET_OFFSET);
82
83 val = ioread32(rng_base + RBG_SOFT_RESET_OFFSET);
84 val &= ~RBG_SOFT_RESET;
85 iowrite32(val, rng_base + RBG_SOFT_RESET_OFFSET);
86
87 /* Enable RBG */
88 val = ioread32(rng_base + RNG_CTRL_OFFSET);
89 val &= ~RNG_CTRL_RNG_RBGEN_MASK;
90 val |= RNG_CTRL_RNG_RBGEN_ENABLE;
91 iowrite32(val, rng_base + RNG_CTRL_OFFSET);
92}
93
94static int iproc_rng200_read(struct hwrng *rng, void *buf, size_t max,
95 bool wait)
96{
97 struct iproc_rng200_dev *priv = to_rng_priv(rng);
98 uint32_t num_remaining = max;
99 uint32_t status;
100
101 #define MAX_RESETS_PER_READ 1
102 uint32_t num_resets = 0;
103
104 #define MAX_IDLE_TIME (1 * HZ)
105 unsigned long idle_endtime = jiffies + MAX_IDLE_TIME;
106
107 while ((num_remaining > 0) && time_before(jiffies, idle_endtime)) {
108
109 /* Is RNG sane? If not, reset it. */
110 status = ioread32(priv->base + RNG_INT_STATUS_OFFSET);
111 if ((status & (RNG_INT_STATUS_MASTER_FAIL_LOCKOUT_IRQ_MASK |
112 RNG_INT_STATUS_NIST_FAIL_IRQ_MASK)) != 0) {
113
114 if (num_resets >= MAX_RESETS_PER_READ)
115 return max - num_remaining;
116
117 iproc_rng200_restart(priv->base);
118 num_resets++;
119 }
120
121 /* Are there any random numbers available? */
122 if ((ioread32(priv->base + RNG_FIFO_COUNT_OFFSET) &
123 RNG_FIFO_COUNT_RNG_FIFO_COUNT_MASK) > 0) {
124
125 if (num_remaining >= sizeof(uint32_t)) {
126 /* Buffer has room to store entire word */
127 *(uint32_t *)buf = ioread32(priv->base +
128 RNG_FIFO_DATA_OFFSET);
129 buf += sizeof(uint32_t);
130 num_remaining -= sizeof(uint32_t);
131 } else {
132 /* Buffer can only store partial word */
133 uint32_t rnd_number = ioread32(priv->base +
134 RNG_FIFO_DATA_OFFSET);
135 memcpy(buf, &rnd_number, num_remaining);
136 buf += num_remaining;
137 num_remaining = 0;
138 }
139
140 /* Reset the IDLE timeout */
141 idle_endtime = jiffies + MAX_IDLE_TIME;
142 } else {
143 if (!wait)
144 /* Cannot wait, return immediately */
145 return max - num_remaining;
146
147 /* Can wait, give others chance to run */
148 usleep_range(min(num_remaining * 10, 500U), 500);
149 }
150 }
151
152 return max - num_remaining;
153}
154
155static int iproc_rng200_init(struct hwrng *rng)
156{
157 struct iproc_rng200_dev *priv = to_rng_priv(rng);
158 uint32_t val;
159
160 /* Setup RNG. */
161 val = ioread32(priv->base + RNG_CTRL_OFFSET);
162 val &= ~RNG_CTRL_RNG_RBGEN_MASK;
163 val |= RNG_CTRL_RNG_RBGEN_ENABLE;
164 iowrite32(val, priv->base + RNG_CTRL_OFFSET);
165
166 return 0;
167}
168
169static void iproc_rng200_cleanup(struct hwrng *rng)
170{
171 struct iproc_rng200_dev *priv = to_rng_priv(rng);
172 uint32_t val;
173
174 /* Disable RNG hardware */
175 val = ioread32(priv->base + RNG_CTRL_OFFSET);
176 val &= ~RNG_CTRL_RNG_RBGEN_MASK;
177 val |= RNG_CTRL_RNG_RBGEN_DISABLE;
178 iowrite32(val, priv->base + RNG_CTRL_OFFSET);
179}
180
181static int iproc_rng200_probe(struct platform_device *pdev)
182{
183 struct iproc_rng200_dev *priv;
184 struct resource *res;
185 struct device *dev = &pdev->dev;
186 int ret;
187
188 priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
189 if (!priv)
190 return -ENOMEM;
191
192 /* Map peripheral */
193 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
194 if (!res) {
195 dev_err(dev, "failed to get rng resources\n");
196 return -EINVAL;
197 }
198
199 priv->base = devm_ioremap_resource(dev, res);
200 if (IS_ERR(priv->base)) {
201 dev_err(dev, "failed to remap rng regs\n");
202 return PTR_ERR(priv->base);
203 }
204
205 priv->rng.name = "iproc-rng200",
206 priv->rng.read = iproc_rng200_read,
207 priv->rng.init = iproc_rng200_init,
208 priv->rng.cleanup = iproc_rng200_cleanup,
209
210 /* Register driver */
211 ret = devm_hwrng_register(dev, &priv->rng);
212 if (ret) {
213 dev_err(dev, "hwrng registration failed\n");
214 return ret;
215 }
216
217 dev_info(dev, "hwrng registered\n");
218
219 return 0;
220}
221
222static const struct of_device_id iproc_rng200_of_match[] = {
223 { .compatible = "brcm,iproc-rng200", },
224 {},
225};
226MODULE_DEVICE_TABLE(of, iproc_rng200_of_match);
227
228static struct platform_driver iproc_rng200_driver = {
229 .driver = {
230 .name = "iproc-rng200",
231 .of_match_table = iproc_rng200_of_match,
232 },
233 .probe = iproc_rng200_probe,
234};
235module_platform_driver(iproc_rng200_driver);
236
237MODULE_AUTHOR("Broadcom");
238MODULE_DESCRIPTION("iProc RNG200 Random Number Generator driver");
239MODULE_LICENSE("GPL v2");
diff --git a/drivers/char/hw_random/msm-rng.c b/drivers/char/hw_random/msm-rng.c
index cea1c703d62f..96fb986402eb 100644
--- a/drivers/char/hw_random/msm-rng.c
+++ b/drivers/char/hw_random/msm-rng.c
@@ -157,7 +157,7 @@ static int msm_rng_probe(struct platform_device *pdev)
157 rng->hwrng.cleanup = msm_rng_cleanup, 157 rng->hwrng.cleanup = msm_rng_cleanup,
158 rng->hwrng.read = msm_rng_read, 158 rng->hwrng.read = msm_rng_read,
159 159
160 ret = hwrng_register(&rng->hwrng); 160 ret = devm_hwrng_register(&pdev->dev, &rng->hwrng);
161 if (ret) { 161 if (ret) {
162 dev_err(&pdev->dev, "failed to register hwrng\n"); 162 dev_err(&pdev->dev, "failed to register hwrng\n");
163 return ret; 163 return ret;
@@ -166,14 +166,6 @@ static int msm_rng_probe(struct platform_device *pdev)
166 return 0; 166 return 0;
167} 167}
168 168
169static int msm_rng_remove(struct platform_device *pdev)
170{
171 struct msm_rng *rng = platform_get_drvdata(pdev);
172
173 hwrng_unregister(&rng->hwrng);
174 return 0;
175}
176
177static const struct of_device_id msm_rng_of_match[] = { 169static const struct of_device_id msm_rng_of_match[] = {
178 { .compatible = "qcom,prng", }, 170 { .compatible = "qcom,prng", },
179 {} 171 {}
@@ -182,7 +174,6 @@ MODULE_DEVICE_TABLE(of, msm_rng_of_match);
182 174
183static struct platform_driver msm_rng_driver = { 175static struct platform_driver msm_rng_driver = {
184 .probe = msm_rng_probe, 176 .probe = msm_rng_probe,
185 .remove = msm_rng_remove,
186 .driver = { 177 .driver = {
187 .name = KBUILD_MODNAME, 178 .name = KBUILD_MODNAME,
188 .of_match_table = of_match_ptr(msm_rng_of_match), 179 .of_match_table = of_match_ptr(msm_rng_of_match),
diff --git a/drivers/char/hw_random/octeon-rng.c b/drivers/char/hw_random/octeon-rng.c
index be1c3f607398..6234a4a19b56 100644
--- a/drivers/char/hw_random/octeon-rng.c
+++ b/drivers/char/hw_random/octeon-rng.c
@@ -105,7 +105,7 @@ static int octeon_rng_probe(struct platform_device *pdev)
105 return 0; 105 return 0;
106} 106}
107 107
108static int __exit octeon_rng_remove(struct platform_device *pdev) 108static int octeon_rng_remove(struct platform_device *pdev)
109{ 109{
110 struct hwrng *rng = platform_get_drvdata(pdev); 110 struct hwrng *rng = platform_get_drvdata(pdev);
111 111
@@ -119,7 +119,7 @@ static struct platform_driver octeon_rng_driver = {
119 .name = "octeon_rng", 119 .name = "octeon_rng",
120 }, 120 },
121 .probe = octeon_rng_probe, 121 .probe = octeon_rng_probe,
122 .remove = __exit_p(octeon_rng_remove), 122 .remove = octeon_rng_remove,
123}; 123};
124 124
125module_platform_driver(octeon_rng_driver); 125module_platform_driver(octeon_rng_driver);
diff --git a/drivers/char/hw_random/omap-rng.c b/drivers/char/hw_random/omap-rng.c
index d14dcf788f17..8a1432e8bb80 100644
--- a/drivers/char/hw_random/omap-rng.c
+++ b/drivers/char/hw_random/omap-rng.c
@@ -236,7 +236,7 @@ static int omap4_rng_init(struct omap_rng_dev *priv)
236 u32 val; 236 u32 val;
237 237
238 /* Return if RNG is already running. */ 238 /* Return if RNG is already running. */
239 if (omap_rng_read(priv, RNG_CONFIG_REG) & RNG_CONTROL_ENABLE_TRNG_MASK) 239 if (omap_rng_read(priv, RNG_CONTROL_REG) & RNG_CONTROL_ENABLE_TRNG_MASK)
240 return 0; 240 return 0;
241 241
242 val = RNG_CONFIG_MIN_REFIL_CYCLES << RNG_CONFIG_MIN_REFIL_CYCLES_SHIFT; 242 val = RNG_CONFIG_MIN_REFIL_CYCLES << RNG_CONFIG_MIN_REFIL_CYCLES_SHIFT;
@@ -262,7 +262,7 @@ static void omap4_rng_cleanup(struct omap_rng_dev *priv)
262 262
263 val = omap_rng_read(priv, RNG_CONTROL_REG); 263 val = omap_rng_read(priv, RNG_CONTROL_REG);
264 val &= ~RNG_CONTROL_ENABLE_TRNG_MASK; 264 val &= ~RNG_CONTROL_ENABLE_TRNG_MASK;
265 omap_rng_write(priv, RNG_CONFIG_REG, val); 265 omap_rng_write(priv, RNG_CONTROL_REG, val);
266} 266}
267 267
268static irqreturn_t omap4_rng_irq(int irq, void *dev_id) 268static irqreturn_t omap4_rng_irq(int irq, void *dev_id)
@@ -408,7 +408,7 @@ err_ioremap:
408 return ret; 408 return ret;
409} 409}
410 410
411static int __exit omap_rng_remove(struct platform_device *pdev) 411static int omap_rng_remove(struct platform_device *pdev)
412{ 412{
413 struct omap_rng_dev *priv = platform_get_drvdata(pdev); 413 struct omap_rng_dev *priv = platform_get_drvdata(pdev);
414 414
@@ -422,9 +422,7 @@ static int __exit omap_rng_remove(struct platform_device *pdev)
422 return 0; 422 return 0;
423} 423}
424 424
425#ifdef CONFIG_PM_SLEEP 425static int __maybe_unused omap_rng_suspend(struct device *dev)
426
427static int omap_rng_suspend(struct device *dev)
428{ 426{
429 struct omap_rng_dev *priv = dev_get_drvdata(dev); 427 struct omap_rng_dev *priv = dev_get_drvdata(dev);
430 428
@@ -434,7 +432,7 @@ static int omap_rng_suspend(struct device *dev)
434 return 0; 432 return 0;
435} 433}
436 434
437static int omap_rng_resume(struct device *dev) 435static int __maybe_unused omap_rng_resume(struct device *dev)
438{ 436{
439 struct omap_rng_dev *priv = dev_get_drvdata(dev); 437 struct omap_rng_dev *priv = dev_get_drvdata(dev);
440 438
@@ -445,22 +443,15 @@ static int omap_rng_resume(struct device *dev)
445} 443}
446 444
447static SIMPLE_DEV_PM_OPS(omap_rng_pm, omap_rng_suspend, omap_rng_resume); 445static SIMPLE_DEV_PM_OPS(omap_rng_pm, omap_rng_suspend, omap_rng_resume);
448#define OMAP_RNG_PM (&omap_rng_pm)
449
450#else
451
452#define OMAP_RNG_PM NULL
453
454#endif
455 446
456static struct platform_driver omap_rng_driver = { 447static struct platform_driver omap_rng_driver = {
457 .driver = { 448 .driver = {
458 .name = "omap_rng", 449 .name = "omap_rng",
459 .pm = OMAP_RNG_PM, 450 .pm = &omap_rng_pm,
460 .of_match_table = of_match_ptr(omap_rng_of_match), 451 .of_match_table = of_match_ptr(omap_rng_of_match),
461 }, 452 },
462 .probe = omap_rng_probe, 453 .probe = omap_rng_probe,
463 .remove = __exit_p(omap_rng_remove), 454 .remove = omap_rng_remove,
464}; 455};
465 456
466module_platform_driver(omap_rng_driver); 457module_platform_driver(omap_rng_driver);
diff --git a/drivers/char/hw_random/pseries-rng.c b/drivers/char/hw_random/pseries-rng.c
index bcf86f91800a..63ce51d09af1 100644
--- a/drivers/char/hw_random/pseries-rng.c
+++ b/drivers/char/hw_random/pseries-rng.c
@@ -61,13 +61,13 @@ static struct hwrng pseries_rng = {
61 .read = pseries_rng_read, 61 .read = pseries_rng_read,
62}; 62};
63 63
64static int __init pseries_rng_probe(struct vio_dev *dev, 64static int pseries_rng_probe(struct vio_dev *dev,
65 const struct vio_device_id *id) 65 const struct vio_device_id *id)
66{ 66{
67 return hwrng_register(&pseries_rng); 67 return hwrng_register(&pseries_rng);
68} 68}
69 69
70static int __exit pseries_rng_remove(struct vio_dev *dev) 70static int pseries_rng_remove(struct vio_dev *dev)
71{ 71{
72 hwrng_unregister(&pseries_rng); 72 hwrng_unregister(&pseries_rng);
73 return 0; 73 return 0;
diff --git a/drivers/char/hw_random/xgene-rng.c b/drivers/char/hw_random/xgene-rng.c
index 23caa05380a8..c37cf754a985 100644
--- a/drivers/char/hw_random/xgene-rng.c
+++ b/drivers/char/hw_random/xgene-rng.c
@@ -21,6 +21,7 @@
21 * 21 *
22 */ 22 */
23 23
24#include <linux/acpi.h>
24#include <linux/clk.h> 25#include <linux/clk.h>
25#include <linux/delay.h> 26#include <linux/delay.h>
26#include <linux/hw_random.h> 27#include <linux/hw_random.h>
@@ -310,6 +311,14 @@ static int xgene_rng_init(struct hwrng *rng)
310 return 0; 311 return 0;
311} 312}
312 313
314#ifdef CONFIG_ACPI
315static const struct acpi_device_id xgene_rng_acpi_match[] = {
316 { "APMC0D18", },
317 { }
318};
319MODULE_DEVICE_TABLE(acpi, xgene_rng_acpi_match);
320#endif
321
313static struct hwrng xgene_rng_func = { 322static struct hwrng xgene_rng_func = {
314 .name = "xgene-rng", 323 .name = "xgene-rng",
315 .init = xgene_rng_init, 324 .init = xgene_rng_init,
@@ -415,6 +424,7 @@ static struct platform_driver xgene_rng_driver = {
415 .driver = { 424 .driver = {
416 .name = "xgene-rng", 425 .name = "xgene-rng",
417 .of_match_table = xgene_rng_of_match, 426 .of_match_table = xgene_rng_of_match,
427 .acpi_match_table = ACPI_PTR(xgene_rng_acpi_match),
418 }, 428 },
419}; 429};
420 430
diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig
index 2fb0fdfc87df..800bf41718e1 100644
--- a/drivers/crypto/Kconfig
+++ b/drivers/crypto/Kconfig
@@ -391,7 +391,7 @@ config CRYPTO_DEV_ATMEL_SHA
391 391
392config CRYPTO_DEV_CCP 392config CRYPTO_DEV_CCP
393 bool "Support for AMD Cryptographic Coprocessor" 393 bool "Support for AMD Cryptographic Coprocessor"
394 depends on (X86 && PCI) || ARM64 394 depends on ((X86 && PCI) || (ARM64 && (OF_ADDRESS || ACPI))) && HAS_IOMEM
395 default n 395 default n
396 help 396 help
397 The AMD Cryptographic Coprocessor provides hardware support 397 The AMD Cryptographic Coprocessor provides hardware support
@@ -436,4 +436,26 @@ config CRYPTO_DEV_QCE
436 hardware. To compile this driver as a module, choose M here. The 436 hardware. To compile this driver as a module, choose M here. The
437 module will be called qcrypto. 437 module will be called qcrypto.
438 438
439config CRYPTO_DEV_VMX
440 bool "Support for VMX cryptographic acceleration instructions"
441 depends on PPC64
442 default n
443 help
444 Support for VMX cryptographic acceleration instructions.
445
446source "drivers/crypto/vmx/Kconfig"
447
448config CRYPTO_DEV_IMGTEC_HASH
449 depends on MIPS || COMPILE_TEST
450 tristate "Imagination Technologies hardware hash accelerator"
451 select CRYPTO_ALGAPI
452 select CRYPTO_MD5
453 select CRYPTO_SHA1
454 select CRYPTO_SHA256
455 select CRYPTO_HASH
456 help
457 This driver interfaces with the Imagination Technologies
458 hardware hash accelerator. Supporting MD5/SHA1/SHA224/SHA256
459 hashing algorithms.
460
439endif # CRYPTO_HW 461endif # CRYPTO_HW
diff --git a/drivers/crypto/Makefile b/drivers/crypto/Makefile
index 3924f93d5774..fb84be7e6be5 100644
--- a/drivers/crypto/Makefile
+++ b/drivers/crypto/Makefile
@@ -6,6 +6,7 @@ obj-$(CONFIG_CRYPTO_DEV_CCP) += ccp/
6obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM) += caam/ 6obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM) += caam/
7obj-$(CONFIG_CRYPTO_DEV_GEODE) += geode-aes.o 7obj-$(CONFIG_CRYPTO_DEV_GEODE) += geode-aes.o
8obj-$(CONFIG_CRYPTO_DEV_HIFN_795X) += hifn_795x.o 8obj-$(CONFIG_CRYPTO_DEV_HIFN_795X) += hifn_795x.o
9obj-$(CONFIG_CRYPTO_DEV_IMGTEC_HASH) += img-hash.o
9obj-$(CONFIG_CRYPTO_DEV_IXP4XX) += ixp4xx_crypto.o 10obj-$(CONFIG_CRYPTO_DEV_IXP4XX) += ixp4xx_crypto.o
10obj-$(CONFIG_CRYPTO_DEV_MV_CESA) += mv_cesa.o 11obj-$(CONFIG_CRYPTO_DEV_MV_CESA) += mv_cesa.o
11obj-$(CONFIG_CRYPTO_DEV_MXS_DCP) += mxs-dcp.o 12obj-$(CONFIG_CRYPTO_DEV_MXS_DCP) += mxs-dcp.o
@@ -25,3 +26,4 @@ obj-$(CONFIG_CRYPTO_DEV_TALITOS) += talitos.o
25obj-$(CONFIG_CRYPTO_DEV_UX500) += ux500/ 26obj-$(CONFIG_CRYPTO_DEV_UX500) += ux500/
26obj-$(CONFIG_CRYPTO_DEV_QAT) += qat/ 27obj-$(CONFIG_CRYPTO_DEV_QAT) += qat/
27obj-$(CONFIG_CRYPTO_DEV_QCE) += qce/ 28obj-$(CONFIG_CRYPTO_DEV_QCE) += qce/
29obj-$(CONFIG_CRYPTO_DEV_VMX) += vmx/
diff --git a/drivers/crypto/amcc/crypto4xx_core.c b/drivers/crypto/amcc/crypto4xx_core.c
index d02b77150070..3b28e8c3de28 100644
--- a/drivers/crypto/amcc/crypto4xx_core.c
+++ b/drivers/crypto/amcc/crypto4xx_core.c
@@ -1155,7 +1155,7 @@ struct crypto4xx_alg_common crypto4xx_alg[] = {
1155/** 1155/**
1156 * Module Initialization Routine 1156 * Module Initialization Routine
1157 */ 1157 */
1158static int __init crypto4xx_probe(struct platform_device *ofdev) 1158static int crypto4xx_probe(struct platform_device *ofdev)
1159{ 1159{
1160 int rc; 1160 int rc;
1161 struct resource res; 1161 struct resource res;
@@ -1263,7 +1263,7 @@ err_alloc_dev:
1263 return rc; 1263 return rc;
1264} 1264}
1265 1265
1266static int __exit crypto4xx_remove(struct platform_device *ofdev) 1266static int crypto4xx_remove(struct platform_device *ofdev)
1267{ 1267{
1268 struct device *dev = &ofdev->dev; 1268 struct device *dev = &ofdev->dev;
1269 struct crypto4xx_core_device *core_dev = dev_get_drvdata(dev); 1269 struct crypto4xx_core_device *core_dev = dev_get_drvdata(dev);
@@ -1291,7 +1291,7 @@ static struct platform_driver crypto4xx_driver = {
1291 .of_match_table = crypto4xx_match, 1291 .of_match_table = crypto4xx_match,
1292 }, 1292 },
1293 .probe = crypto4xx_probe, 1293 .probe = crypto4xx_probe,
1294 .remove = __exit_p(crypto4xx_remove), 1294 .remove = crypto4xx_remove,
1295}; 1295};
1296 1296
1297module_platform_driver(crypto4xx_driver); 1297module_platform_driver(crypto4xx_driver);
diff --git a/drivers/crypto/atmel-aes.c b/drivers/crypto/atmel-aes.c
index 6597aac9905d..0f9a9dc06a83 100644
--- a/drivers/crypto/atmel-aes.c
+++ b/drivers/crypto/atmel-aes.c
@@ -315,10 +315,10 @@ static int atmel_aes_crypt_dma(struct atmel_aes_dev *dd,
315 315
316 dd->dma_size = length; 316 dd->dma_size = length;
317 317
318 if (!(dd->flags & AES_FLAGS_FAST)) { 318 dma_sync_single_for_device(dd->dev, dma_addr_in, length,
319 dma_sync_single_for_device(dd->dev, dma_addr_in, length, 319 DMA_TO_DEVICE);
320 DMA_TO_DEVICE); 320 dma_sync_single_for_device(dd->dev, dma_addr_out, length,
321 } 321 DMA_FROM_DEVICE);
322 322
323 if (dd->flags & AES_FLAGS_CFB8) { 323 if (dd->flags & AES_FLAGS_CFB8) {
324 dd->dma_lch_in.dma_conf.dst_addr_width = 324 dd->dma_lch_in.dma_conf.dst_addr_width =
@@ -391,6 +391,11 @@ static int atmel_aes_crypt_cpu_start(struct atmel_aes_dev *dd)
391{ 391{
392 dd->flags &= ~AES_FLAGS_DMA; 392 dd->flags &= ~AES_FLAGS_DMA;
393 393
394 dma_sync_single_for_cpu(dd->dev, dd->dma_addr_in,
395 dd->dma_size, DMA_TO_DEVICE);
396 dma_sync_single_for_cpu(dd->dev, dd->dma_addr_out,
397 dd->dma_size, DMA_FROM_DEVICE);
398
394 /* use cache buffers */ 399 /* use cache buffers */
395 dd->nb_in_sg = atmel_aes_sg_length(dd->req, dd->in_sg); 400 dd->nb_in_sg = atmel_aes_sg_length(dd->req, dd->in_sg);
396 if (!dd->nb_in_sg) 401 if (!dd->nb_in_sg)
@@ -459,6 +464,9 @@ static int atmel_aes_crypt_dma_start(struct atmel_aes_dev *dd)
459 dd->flags |= AES_FLAGS_FAST; 464 dd->flags |= AES_FLAGS_FAST;
460 465
461 } else { 466 } else {
467 dma_sync_single_for_cpu(dd->dev, dd->dma_addr_in,
468 dd->dma_size, DMA_TO_DEVICE);
469
462 /* use cache buffers */ 470 /* use cache buffers */
463 count = atmel_aes_sg_copy(&dd->in_sg, &dd->in_offset, 471 count = atmel_aes_sg_copy(&dd->in_sg, &dd->in_offset,
464 dd->buf_in, dd->buflen, dd->total, 0); 472 dd->buf_in, dd->buflen, dd->total, 0);
@@ -619,7 +627,7 @@ static int atmel_aes_crypt_dma_stop(struct atmel_aes_dev *dd)
619 dma_unmap_sg(dd->dev, dd->out_sg, 1, DMA_FROM_DEVICE); 627 dma_unmap_sg(dd->dev, dd->out_sg, 1, DMA_FROM_DEVICE);
620 dma_unmap_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE); 628 dma_unmap_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE);
621 } else { 629 } else {
622 dma_sync_single_for_device(dd->dev, dd->dma_addr_out, 630 dma_sync_single_for_cpu(dd->dev, dd->dma_addr_out,
623 dd->dma_size, DMA_FROM_DEVICE); 631 dd->dma_size, DMA_FROM_DEVICE);
624 632
625 /* copy data */ 633 /* copy data */
@@ -1246,6 +1254,11 @@ static void atmel_aes_get_cap(struct atmel_aes_dev *dd)
1246 1254
1247 /* keep only major version number */ 1255 /* keep only major version number */
1248 switch (dd->hw_version & 0xff0) { 1256 switch (dd->hw_version & 0xff0) {
1257 case 0x200:
1258 dd->caps.has_dualbuff = 1;
1259 dd->caps.has_cfb64 = 1;
1260 dd->caps.max_burst_size = 4;
1261 break;
1249 case 0x130: 1262 case 0x130:
1250 dd->caps.has_dualbuff = 1; 1263 dd->caps.has_dualbuff = 1;
1251 dd->caps.has_cfb64 = 1; 1264 dd->caps.has_cfb64 = 1;
@@ -1336,6 +1349,7 @@ static int atmel_aes_probe(struct platform_device *pdev)
1336 platform_set_drvdata(pdev, aes_dd); 1349 platform_set_drvdata(pdev, aes_dd);
1337 1350
1338 INIT_LIST_HEAD(&aes_dd->list); 1351 INIT_LIST_HEAD(&aes_dd->list);
1352 spin_lock_init(&aes_dd->lock);
1339 1353
1340 tasklet_init(&aes_dd->done_task, atmel_aes_done_task, 1354 tasklet_init(&aes_dd->done_task, atmel_aes_done_task,
1341 (unsigned long)aes_dd); 1355 (unsigned long)aes_dd);
@@ -1374,7 +1388,7 @@ static int atmel_aes_probe(struct platform_device *pdev)
1374 /* Initializing the clock */ 1388 /* Initializing the clock */
1375 aes_dd->iclk = clk_get(&pdev->dev, "aes_clk"); 1389 aes_dd->iclk = clk_get(&pdev->dev, "aes_clk");
1376 if (IS_ERR(aes_dd->iclk)) { 1390 if (IS_ERR(aes_dd->iclk)) {
1377 dev_err(dev, "clock intialization failed.\n"); 1391 dev_err(dev, "clock initialization failed.\n");
1378 err = PTR_ERR(aes_dd->iclk); 1392 err = PTR_ERR(aes_dd->iclk);
1379 goto clk_err; 1393 goto clk_err;
1380 } 1394 }
diff --git a/drivers/crypto/atmel-sha.c b/drivers/crypto/atmel-sha.c
index 34db04addc18..5b35433c5399 100644
--- a/drivers/crypto/atmel-sha.c
+++ b/drivers/crypto/atmel-sha.c
@@ -163,8 +163,20 @@ static size_t atmel_sha_append_sg(struct atmel_sha_reqctx *ctx)
163 count = min(ctx->sg->length - ctx->offset, ctx->total); 163 count = min(ctx->sg->length - ctx->offset, ctx->total);
164 count = min(count, ctx->buflen - ctx->bufcnt); 164 count = min(count, ctx->buflen - ctx->bufcnt);
165 165
166 if (count <= 0) 166 if (count <= 0) {
167 break; 167 /*
168 * Check if count <= 0 because the buffer is full or
169 * because the sg length is 0. In the latest case,
170 * check if there is another sg in the list, a 0 length
171 * sg doesn't necessarily mean the end of the sg list.
172 */
173 if ((ctx->sg->length == 0) && !sg_is_last(ctx->sg)) {
174 ctx->sg = sg_next(ctx->sg);
175 continue;
176 } else {
177 break;
178 }
179 }
168 180
169 scatterwalk_map_and_copy(ctx->buffer + ctx->bufcnt, ctx->sg, 181 scatterwalk_map_and_copy(ctx->buffer + ctx->bufcnt, ctx->sg,
170 ctx->offset, count, 0); 182 ctx->offset, count, 0);
@@ -420,14 +432,8 @@ static int atmel_sha_xmit_dma(struct atmel_sha_dev *dd, dma_addr_t dma_addr1,
420 dev_dbg(dd->dev, "xmit_dma: digcnt: 0x%llx 0x%llx, length: %d, final: %d\n", 432 dev_dbg(dd->dev, "xmit_dma: digcnt: 0x%llx 0x%llx, length: %d, final: %d\n",
421 ctx->digcnt[1], ctx->digcnt[0], length1, final); 433 ctx->digcnt[1], ctx->digcnt[0], length1, final);
422 434
423 if (ctx->flags & (SHA_FLAGS_SHA1 | SHA_FLAGS_SHA224 | 435 dd->dma_lch_in.dma_conf.src_maxburst = 16;
424 SHA_FLAGS_SHA256)) { 436 dd->dma_lch_in.dma_conf.dst_maxburst = 16;
425 dd->dma_lch_in.dma_conf.src_maxburst = 16;
426 dd->dma_lch_in.dma_conf.dst_maxburst = 16;
427 } else {
428 dd->dma_lch_in.dma_conf.src_maxburst = 32;
429 dd->dma_lch_in.dma_conf.dst_maxburst = 32;
430 }
431 437
432 dmaengine_slave_config(dd->dma_lch_in.chan, &dd->dma_lch_in.dma_conf); 438 dmaengine_slave_config(dd->dma_lch_in.chan, &dd->dma_lch_in.dma_conf);
433 439
@@ -529,7 +535,7 @@ static int atmel_sha_update_dma_slow(struct atmel_sha_dev *dd)
529 if (final) 535 if (final)
530 atmel_sha_fill_padding(ctx, 0); 536 atmel_sha_fill_padding(ctx, 0);
531 537
532 if (final || (ctx->bufcnt == ctx->buflen && ctx->total)) { 538 if (final || (ctx->bufcnt == ctx->buflen)) {
533 count = ctx->bufcnt; 539 count = ctx->bufcnt;
534 ctx->bufcnt = 0; 540 ctx->bufcnt = 0;
535 return atmel_sha_xmit_dma_map(dd, ctx, count, final); 541 return atmel_sha_xmit_dma_map(dd, ctx, count, final);
@@ -1266,6 +1272,12 @@ static void atmel_sha_get_cap(struct atmel_sha_dev *dd)
1266 1272
1267 /* keep only major version number */ 1273 /* keep only major version number */
1268 switch (dd->hw_version & 0xff0) { 1274 switch (dd->hw_version & 0xff0) {
1275 case 0x420:
1276 dd->caps.has_dma = 1;
1277 dd->caps.has_dualbuff = 1;
1278 dd->caps.has_sha224 = 1;
1279 dd->caps.has_sha_384_512 = 1;
1280 break;
1269 case 0x410: 1281 case 0x410:
1270 dd->caps.has_dma = 1; 1282 dd->caps.has_dma = 1;
1271 dd->caps.has_dualbuff = 1; 1283 dd->caps.has_dualbuff = 1;
@@ -1349,6 +1361,7 @@ static int atmel_sha_probe(struct platform_device *pdev)
1349 platform_set_drvdata(pdev, sha_dd); 1361 platform_set_drvdata(pdev, sha_dd);
1350 1362
1351 INIT_LIST_HEAD(&sha_dd->list); 1363 INIT_LIST_HEAD(&sha_dd->list);
1364 spin_lock_init(&sha_dd->lock);
1352 1365
1353 tasklet_init(&sha_dd->done_task, atmel_sha_done_task, 1366 tasklet_init(&sha_dd->done_task, atmel_sha_done_task,
1354 (unsigned long)sha_dd); 1367 (unsigned long)sha_dd);
@@ -1385,7 +1398,7 @@ static int atmel_sha_probe(struct platform_device *pdev)
1385 /* Initializing the clock */ 1398 /* Initializing the clock */
1386 sha_dd->iclk = clk_get(&pdev->dev, "sha_clk"); 1399 sha_dd->iclk = clk_get(&pdev->dev, "sha_clk");
1387 if (IS_ERR(sha_dd->iclk)) { 1400 if (IS_ERR(sha_dd->iclk)) {
1388 dev_err(dev, "clock intialization failed.\n"); 1401 dev_err(dev, "clock initialization failed.\n");
1389 err = PTR_ERR(sha_dd->iclk); 1402 err = PTR_ERR(sha_dd->iclk);
1390 goto clk_err; 1403 goto clk_err;
1391 } 1404 }
diff --git a/drivers/crypto/atmel-tdes.c b/drivers/crypto/atmel-tdes.c
index 258772d9b22f..ca2999709eb4 100644
--- a/drivers/crypto/atmel-tdes.c
+++ b/drivers/crypto/atmel-tdes.c
@@ -1370,6 +1370,7 @@ static int atmel_tdes_probe(struct platform_device *pdev)
1370 platform_set_drvdata(pdev, tdes_dd); 1370 platform_set_drvdata(pdev, tdes_dd);
1371 1371
1372 INIT_LIST_HEAD(&tdes_dd->list); 1372 INIT_LIST_HEAD(&tdes_dd->list);
1373 spin_lock_init(&tdes_dd->lock);
1373 1374
1374 tasklet_init(&tdes_dd->done_task, atmel_tdes_done_task, 1375 tasklet_init(&tdes_dd->done_task, atmel_tdes_done_task,
1375 (unsigned long)tdes_dd); 1376 (unsigned long)tdes_dd);
@@ -1408,7 +1409,7 @@ static int atmel_tdes_probe(struct platform_device *pdev)
1408 /* Initializing the clock */ 1409 /* Initializing the clock */
1409 tdes_dd->iclk = clk_get(&pdev->dev, "tdes_clk"); 1410 tdes_dd->iclk = clk_get(&pdev->dev, "tdes_clk");
1410 if (IS_ERR(tdes_dd->iclk)) { 1411 if (IS_ERR(tdes_dd->iclk)) {
1411 dev_err(dev, "clock intialization failed.\n"); 1412 dev_err(dev, "clock initialization failed.\n");
1412 err = PTR_ERR(tdes_dd->iclk); 1413 err = PTR_ERR(tdes_dd->iclk);
1413 goto clk_err; 1414 goto clk_err;
1414 } 1415 }
diff --git a/drivers/crypto/caam/caamhash.c b/drivers/crypto/caam/caamhash.c
index f347ab7eea95..ba0532efd3ae 100644
--- a/drivers/crypto/caam/caamhash.c
+++ b/drivers/crypto/caam/caamhash.c
@@ -1172,6 +1172,7 @@ static int ahash_final_no_ctx(struct ahash_request *req)
1172 return -ENOMEM; 1172 return -ENOMEM;
1173 } 1173 }
1174 1174
1175 edesc->sec4_sg_bytes = 0;
1175 sh_len = desc_len(sh_desc); 1176 sh_len = desc_len(sh_desc);
1176 desc = edesc->hw_desc; 1177 desc = edesc->hw_desc;
1177 init_job_desc_shared(desc, ptr, sh_len, HDR_SHARE_DEFER | HDR_REVERSE); 1178 init_job_desc_shared(desc, ptr, sh_len, HDR_SHARE_DEFER | HDR_REVERSE);
diff --git a/drivers/crypto/caam/caamrng.c b/drivers/crypto/caam/caamrng.c
index ae31e555793c..26a544b505f1 100644
--- a/drivers/crypto/caam/caamrng.c
+++ b/drivers/crypto/caam/caamrng.c
@@ -52,7 +52,7 @@
52 52
53/* length of descriptors */ 53/* length of descriptors */
54#define DESC_JOB_O_LEN (CAAM_CMD_SZ * 2 + CAAM_PTR_SZ * 2) 54#define DESC_JOB_O_LEN (CAAM_CMD_SZ * 2 + CAAM_PTR_SZ * 2)
55#define DESC_RNG_LEN (10 * CAAM_CMD_SZ) 55#define DESC_RNG_LEN (4 * CAAM_CMD_SZ)
56 56
57/* Buffer, its dma address and lock */ 57/* Buffer, its dma address and lock */
58struct buf_data { 58struct buf_data {
@@ -90,8 +90,8 @@ static inline void rng_unmap_ctx(struct caam_rng_ctx *ctx)
90 struct device *jrdev = ctx->jrdev; 90 struct device *jrdev = ctx->jrdev;
91 91
92 if (ctx->sh_desc_dma) 92 if (ctx->sh_desc_dma)
93 dma_unmap_single(jrdev, ctx->sh_desc_dma, DESC_RNG_LEN, 93 dma_unmap_single(jrdev, ctx->sh_desc_dma,
94 DMA_TO_DEVICE); 94 desc_bytes(ctx->sh_desc), DMA_TO_DEVICE);
95 rng_unmap_buf(jrdev, &ctx->bufs[0]); 95 rng_unmap_buf(jrdev, &ctx->bufs[0]);
96 rng_unmap_buf(jrdev, &ctx->bufs[1]); 96 rng_unmap_buf(jrdev, &ctx->bufs[1]);
97} 97}
diff --git a/drivers/crypto/ccp/Makefile b/drivers/crypto/ccp/Makefile
index 7f592d8d07bb..55a1f3951578 100644
--- a/drivers/crypto/ccp/Makefile
+++ b/drivers/crypto/ccp/Makefile
@@ -1,11 +1,6 @@
1obj-$(CONFIG_CRYPTO_DEV_CCP_DD) += ccp.o 1obj-$(CONFIG_CRYPTO_DEV_CCP_DD) += ccp.o
2ccp-objs := ccp-dev.o ccp-ops.o 2ccp-objs := ccp-dev.o ccp-ops.o ccp-platform.o
3ifdef CONFIG_X86 3ccp-$(CONFIG_PCI) += ccp-pci.o
4ccp-objs += ccp-pci.o
5endif
6ifdef CONFIG_ARM64
7ccp-objs += ccp-platform.o
8endif
9 4
10obj-$(CONFIG_CRYPTO_DEV_CCP_CRYPTO) += ccp-crypto.o 5obj-$(CONFIG_CRYPTO_DEV_CCP_CRYPTO) += ccp-crypto.o
11ccp-crypto-objs := ccp-crypto-main.o \ 6ccp-crypto-objs := ccp-crypto-main.o \
diff --git a/drivers/crypto/ccp/ccp-crypto-aes-cmac.c b/drivers/crypto/ccp/ccp-crypto-aes-cmac.c
index 8e162ad82085..ea7e8446956a 100644
--- a/drivers/crypto/ccp/ccp-crypto-aes-cmac.c
+++ b/drivers/crypto/ccp/ccp-crypto-aes-cmac.c
@@ -23,7 +23,6 @@
23 23
24#include "ccp-crypto.h" 24#include "ccp-crypto.h"
25 25
26
27static int ccp_aes_cmac_complete(struct crypto_async_request *async_req, 26static int ccp_aes_cmac_complete(struct crypto_async_request *async_req,
28 int ret) 27 int ret)
29{ 28{
@@ -38,11 +37,13 @@ static int ccp_aes_cmac_complete(struct crypto_async_request *async_req,
38 if (rctx->hash_rem) { 37 if (rctx->hash_rem) {
39 /* Save remaining data to buffer */ 38 /* Save remaining data to buffer */
40 unsigned int offset = rctx->nbytes - rctx->hash_rem; 39 unsigned int offset = rctx->nbytes - rctx->hash_rem;
40
41 scatterwalk_map_and_copy(rctx->buf, rctx->src, 41 scatterwalk_map_and_copy(rctx->buf, rctx->src,
42 offset, rctx->hash_rem, 0); 42 offset, rctx->hash_rem, 0);
43 rctx->buf_count = rctx->hash_rem; 43 rctx->buf_count = rctx->hash_rem;
44 } else 44 } else {
45 rctx->buf_count = 0; 45 rctx->buf_count = 0;
46 }
46 47
47 /* Update result area if supplied */ 48 /* Update result area if supplied */
48 if (req->result) 49 if (req->result)
@@ -202,7 +203,7 @@ static int ccp_aes_cmac_digest(struct ahash_request *req)
202} 203}
203 204
204static int ccp_aes_cmac_setkey(struct crypto_ahash *tfm, const u8 *key, 205static int ccp_aes_cmac_setkey(struct crypto_ahash *tfm, const u8 *key,
205 unsigned int key_len) 206 unsigned int key_len)
206{ 207{
207 struct ccp_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm)); 208 struct ccp_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
208 struct ccp_crypto_ahash_alg *alg = 209 struct ccp_crypto_ahash_alg *alg =
@@ -292,7 +293,8 @@ static int ccp_aes_cmac_cra_init(struct crypto_tfm *tfm)
292 crypto_ahash_set_reqsize(ahash, sizeof(struct ccp_aes_cmac_req_ctx)); 293 crypto_ahash_set_reqsize(ahash, sizeof(struct ccp_aes_cmac_req_ctx));
293 294
294 cipher_tfm = crypto_alloc_cipher("aes", 0, 295 cipher_tfm = crypto_alloc_cipher("aes", 0,
295 CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK); 296 CRYPTO_ALG_ASYNC |
297 CRYPTO_ALG_NEED_FALLBACK);
296 if (IS_ERR(cipher_tfm)) { 298 if (IS_ERR(cipher_tfm)) {
297 pr_warn("could not load aes cipher driver\n"); 299 pr_warn("could not load aes cipher driver\n");
298 return PTR_ERR(cipher_tfm); 300 return PTR_ERR(cipher_tfm);
@@ -354,7 +356,7 @@ int ccp_register_aes_cmac_algs(struct list_head *head)
354 ret = crypto_register_ahash(alg); 356 ret = crypto_register_ahash(alg);
355 if (ret) { 357 if (ret) {
356 pr_err("%s ahash algorithm registration error (%d)\n", 358 pr_err("%s ahash algorithm registration error (%d)\n",
357 base->cra_name, ret); 359 base->cra_name, ret);
358 kfree(ccp_alg); 360 kfree(ccp_alg);
359 return ret; 361 return ret;
360 } 362 }
diff --git a/drivers/crypto/ccp/ccp-crypto-aes-xts.c b/drivers/crypto/ccp/ccp-crypto-aes-xts.c
index 0cc5594b7de3..52c7395cb8d8 100644
--- a/drivers/crypto/ccp/ccp-crypto-aes-xts.c
+++ b/drivers/crypto/ccp/ccp-crypto-aes-xts.c
@@ -21,7 +21,6 @@
21 21
22#include "ccp-crypto.h" 22#include "ccp-crypto.h"
23 23
24
25struct ccp_aes_xts_def { 24struct ccp_aes_xts_def {
26 const char *name; 25 const char *name;
27 const char *drv_name; 26 const char *drv_name;
@@ -216,7 +215,6 @@ static void ccp_aes_xts_cra_exit(struct crypto_tfm *tfm)
216 ctx->u.aes.tfm_ablkcipher = NULL; 215 ctx->u.aes.tfm_ablkcipher = NULL;
217} 216}
218 217
219
220static int ccp_register_aes_xts_alg(struct list_head *head, 218static int ccp_register_aes_xts_alg(struct list_head *head,
221 const struct ccp_aes_xts_def *def) 219 const struct ccp_aes_xts_def *def)
222{ 220{
@@ -255,7 +253,7 @@ static int ccp_register_aes_xts_alg(struct list_head *head,
255 ret = crypto_register_alg(alg); 253 ret = crypto_register_alg(alg);
256 if (ret) { 254 if (ret) {
257 pr_err("%s ablkcipher algorithm registration error (%d)\n", 255 pr_err("%s ablkcipher algorithm registration error (%d)\n",
258 alg->cra_name, ret); 256 alg->cra_name, ret);
259 kfree(ccp_alg); 257 kfree(ccp_alg);
260 return ret; 258 return ret;
261 } 259 }
diff --git a/drivers/crypto/ccp/ccp-crypto-aes.c b/drivers/crypto/ccp/ccp-crypto-aes.c
index e46490db0f63..7984f910884d 100644
--- a/drivers/crypto/ccp/ccp-crypto-aes.c
+++ b/drivers/crypto/ccp/ccp-crypto-aes.c
@@ -22,7 +22,6 @@
22 22
23#include "ccp-crypto.h" 23#include "ccp-crypto.h"
24 24
25
26static int ccp_aes_complete(struct crypto_async_request *async_req, int ret) 25static int ccp_aes_complete(struct crypto_async_request *async_req, int ret)
27{ 26{
28 struct ablkcipher_request *req = ablkcipher_request_cast(async_req); 27 struct ablkcipher_request *req = ablkcipher_request_cast(async_req);
@@ -345,7 +344,7 @@ static int ccp_register_aes_alg(struct list_head *head,
345 ret = crypto_register_alg(alg); 344 ret = crypto_register_alg(alg);
346 if (ret) { 345 if (ret) {
347 pr_err("%s ablkcipher algorithm registration error (%d)\n", 346 pr_err("%s ablkcipher algorithm registration error (%d)\n",
348 alg->cra_name, ret); 347 alg->cra_name, ret);
349 kfree(ccp_alg); 348 kfree(ccp_alg);
350 return ret; 349 return ret;
351 } 350 }
diff --git a/drivers/crypto/ccp/ccp-crypto-main.c b/drivers/crypto/ccp/ccp-crypto-main.c
index 4d4e016d755b..bdec01ec608f 100644
--- a/drivers/crypto/ccp/ccp-crypto-main.c
+++ b/drivers/crypto/ccp/ccp-crypto-main.c
@@ -33,7 +33,6 @@ static unsigned int sha_disable;
33module_param(sha_disable, uint, 0444); 33module_param(sha_disable, uint, 0444);
34MODULE_PARM_DESC(sha_disable, "Disable use of SHA - any non-zero value"); 34MODULE_PARM_DESC(sha_disable, "Disable use of SHA - any non-zero value");
35 35
36
37/* List heads for the supported algorithms */ 36/* List heads for the supported algorithms */
38static LIST_HEAD(hash_algs); 37static LIST_HEAD(hash_algs);
39static LIST_HEAD(cipher_algs); 38static LIST_HEAD(cipher_algs);
@@ -48,6 +47,7 @@ struct ccp_crypto_queue {
48 struct list_head *backlog; 47 struct list_head *backlog;
49 unsigned int cmd_count; 48 unsigned int cmd_count;
50}; 49};
50
51#define CCP_CRYPTO_MAX_QLEN 100 51#define CCP_CRYPTO_MAX_QLEN 100
52 52
53static struct ccp_crypto_queue req_queue; 53static struct ccp_crypto_queue req_queue;
@@ -77,7 +77,6 @@ struct ccp_crypto_cpu {
77 int err; 77 int err;
78}; 78};
79 79
80
81static inline bool ccp_crypto_success(int err) 80static inline bool ccp_crypto_success(int err)
82{ 81{
83 if (err && (err != -EINPROGRESS) && (err != -EBUSY)) 82 if (err && (err != -EINPROGRESS) && (err != -EBUSY))
@@ -143,7 +142,7 @@ static void ccp_crypto_complete(void *data, int err)
143 int ret; 142 int ret;
144 143
145 if (err == -EINPROGRESS) { 144 if (err == -EINPROGRESS) {
146 /* Only propogate the -EINPROGRESS if necessary */ 145 /* Only propagate the -EINPROGRESS if necessary */
147 if (crypto_cmd->ret == -EBUSY) { 146 if (crypto_cmd->ret == -EBUSY) {
148 crypto_cmd->ret = -EINPROGRESS; 147 crypto_cmd->ret = -EINPROGRESS;
149 req->complete(req, -EINPROGRESS); 148 req->complete(req, -EINPROGRESS);
diff --git a/drivers/crypto/ccp/ccp-crypto-sha.c b/drivers/crypto/ccp/ccp-crypto-sha.c
index 96531571f7cf..507b34e0cc19 100644
--- a/drivers/crypto/ccp/ccp-crypto-sha.c
+++ b/drivers/crypto/ccp/ccp-crypto-sha.c
@@ -23,7 +23,6 @@
23 23
24#include "ccp-crypto.h" 24#include "ccp-crypto.h"
25 25
26
27static int ccp_sha_complete(struct crypto_async_request *async_req, int ret) 26static int ccp_sha_complete(struct crypto_async_request *async_req, int ret)
28{ 27{
29 struct ahash_request *req = ahash_request_cast(async_req); 28 struct ahash_request *req = ahash_request_cast(async_req);
@@ -37,11 +36,13 @@ static int ccp_sha_complete(struct crypto_async_request *async_req, int ret)
37 if (rctx->hash_rem) { 36 if (rctx->hash_rem) {
38 /* Save remaining data to buffer */ 37 /* Save remaining data to buffer */
39 unsigned int offset = rctx->nbytes - rctx->hash_rem; 38 unsigned int offset = rctx->nbytes - rctx->hash_rem;
39
40 scatterwalk_map_and_copy(rctx->buf, rctx->src, 40 scatterwalk_map_and_copy(rctx->buf, rctx->src,
41 offset, rctx->hash_rem, 0); 41 offset, rctx->hash_rem, 0);
42 rctx->buf_count = rctx->hash_rem; 42 rctx->buf_count = rctx->hash_rem;
43 } else 43 } else {
44 rctx->buf_count = 0; 44 rctx->buf_count = 0;
45 }
45 46
46 /* Update result area if supplied */ 47 /* Update result area if supplied */
47 if (req->result) 48 if (req->result)
@@ -227,8 +228,9 @@ static int ccp_sha_setkey(struct crypto_ahash *tfm, const u8 *key,
227 } 228 }
228 229
229 key_len = digest_size; 230 key_len = digest_size;
230 } else 231 } else {
231 memcpy(ctx->u.sha.key, key, key_len); 232 memcpy(ctx->u.sha.key, key, key_len);
233 }
232 234
233 for (i = 0; i < block_size; i++) { 235 for (i = 0; i < block_size; i++) {
234 ctx->u.sha.ipad[i] = ctx->u.sha.key[i] ^ 0x36; 236 ctx->u.sha.ipad[i] = ctx->u.sha.key[i] ^ 0x36;
@@ -355,7 +357,7 @@ static int ccp_register_hmac_alg(struct list_head *head,
355 ret = crypto_register_ahash(alg); 357 ret = crypto_register_ahash(alg);
356 if (ret) { 358 if (ret) {
357 pr_err("%s ahash algorithm registration error (%d)\n", 359 pr_err("%s ahash algorithm registration error (%d)\n",
358 base->cra_name, ret); 360 base->cra_name, ret);
359 kfree(ccp_alg); 361 kfree(ccp_alg);
360 return ret; 362 return ret;
361 } 363 }
@@ -410,7 +412,7 @@ static int ccp_register_sha_alg(struct list_head *head,
410 ret = crypto_register_ahash(alg); 412 ret = crypto_register_ahash(alg);
411 if (ret) { 413 if (ret) {
412 pr_err("%s ahash algorithm registration error (%d)\n", 414 pr_err("%s ahash algorithm registration error (%d)\n",
413 base->cra_name, ret); 415 base->cra_name, ret);
414 kfree(ccp_alg); 416 kfree(ccp_alg);
415 return ret; 417 return ret;
416 } 418 }
diff --git a/drivers/crypto/ccp/ccp-crypto.h b/drivers/crypto/ccp/ccp-crypto.h
index 9aa4ae184f7f..76a96f0f44c6 100644
--- a/drivers/crypto/ccp/ccp-crypto.h
+++ b/drivers/crypto/ccp/ccp-crypto.h
@@ -13,7 +13,6 @@
13#ifndef __CCP_CRYPTO_H__ 13#ifndef __CCP_CRYPTO_H__
14#define __CCP_CRYPTO_H__ 14#define __CCP_CRYPTO_H__
15 15
16
17#include <linux/list.h> 16#include <linux/list.h>
18#include <linux/wait.h> 17#include <linux/wait.h>
19#include <linux/pci.h> 18#include <linux/pci.h>
@@ -25,7 +24,6 @@
25#include <crypto/hash.h> 24#include <crypto/hash.h>
26#include <crypto/sha.h> 25#include <crypto/sha.h>
27 26
28
29#define CCP_CRA_PRIORITY 300 27#define CCP_CRA_PRIORITY 300
30 28
31struct ccp_crypto_ablkcipher_alg { 29struct ccp_crypto_ablkcipher_alg {
@@ -68,7 +66,6 @@ static inline struct ccp_crypto_ahash_alg *
68 return container_of(ahash_alg, struct ccp_crypto_ahash_alg, alg); 66 return container_of(ahash_alg, struct ccp_crypto_ahash_alg, alg);
69} 67}
70 68
71
72/***** AES related defines *****/ 69/***** AES related defines *****/
73struct ccp_aes_ctx { 70struct ccp_aes_ctx {
74 /* Fallback cipher for XTS with unsupported unit sizes */ 71 /* Fallback cipher for XTS with unsupported unit sizes */
diff --git a/drivers/crypto/ccp/ccp-dev.c b/drivers/crypto/ccp/ccp-dev.c
index ca29c120b85f..861bacc1bb94 100644
--- a/drivers/crypto/ccp/ccp-dev.c
+++ b/drivers/crypto/ccp/ccp-dev.c
@@ -37,7 +37,6 @@ struct ccp_tasklet_data {
37 struct ccp_cmd *cmd; 37 struct ccp_cmd *cmd;
38}; 38};
39 39
40
41static struct ccp_device *ccp_dev; 40static struct ccp_device *ccp_dev;
42static inline struct ccp_device *ccp_get_device(void) 41static inline struct ccp_device *ccp_get_device(void)
43{ 42{
@@ -296,11 +295,9 @@ struct ccp_device *ccp_alloc_struct(struct device *dev)
296{ 295{
297 struct ccp_device *ccp; 296 struct ccp_device *ccp;
298 297
299 ccp = kzalloc(sizeof(*ccp), GFP_KERNEL); 298 ccp = devm_kzalloc(dev, sizeof(*ccp), GFP_KERNEL);
300 if (ccp == NULL) { 299 if (!ccp)
301 dev_err(dev, "unable to allocate device struct\n");
302 return NULL; 300 return NULL;
303 }
304 ccp->dev = dev; 301 ccp->dev = dev;
305 302
306 INIT_LIST_HEAD(&ccp->cmd); 303 INIT_LIST_HEAD(&ccp->cmd);
diff --git a/drivers/crypto/ccp/ccp-dev.h b/drivers/crypto/ccp/ccp-dev.h
index 62ff35a6b9ec..6ff89031fb96 100644
--- a/drivers/crypto/ccp/ccp-dev.h
+++ b/drivers/crypto/ccp/ccp-dev.h
@@ -21,7 +21,7 @@
21#include <linux/wait.h> 21#include <linux/wait.h>
22#include <linux/dmapool.h> 22#include <linux/dmapool.h>
23#include <linux/hw_random.h> 23#include <linux/hw_random.h>
24 24#include <linux/bitops.h>
25 25
26#define MAX_DMAPOOL_NAME_LEN 32 26#define MAX_DMAPOOL_NAME_LEN 32
27 27
@@ -33,7 +33,6 @@
33#define CACHE_NONE 0x00 33#define CACHE_NONE 0x00
34#define CACHE_WB_NO_ALLOC 0xb7 34#define CACHE_WB_NO_ALLOC 0xb7
35 35
36
37/****** Register Mappings ******/ 36/****** Register Mappings ******/
38#define Q_MASK_REG 0x000 37#define Q_MASK_REG 0x000
39#define TRNG_OUT_REG 0x00c 38#define TRNG_OUT_REG 0x00c
@@ -54,8 +53,8 @@
54#define CMD_Q_CACHE_BASE 0x228 53#define CMD_Q_CACHE_BASE 0x228
55#define CMD_Q_CACHE_INC 0x20 54#define CMD_Q_CACHE_INC 0x20
56 55
57#define CMD_Q_ERROR(__qs) ((__qs) & 0x0000003f); 56#define CMD_Q_ERROR(__qs) ((__qs) & 0x0000003f)
58#define CMD_Q_DEPTH(__qs) (((__qs) >> 12) & 0x0000000f); 57#define CMD_Q_DEPTH(__qs) (((__qs) >> 12) & 0x0000000f)
59 58
60/****** REQ0 Related Values ******/ 59/****** REQ0 Related Values ******/
61#define REQ0_WAIT_FOR_WRITE 0x00000004 60#define REQ0_WAIT_FOR_WRITE 0x00000004
@@ -103,7 +102,6 @@
103/****** REQ6 Related Values ******/ 102/****** REQ6 Related Values ******/
104#define REQ6_MEMTYPE_SHIFT 16 103#define REQ6_MEMTYPE_SHIFT 16
105 104
106
107/****** Key Storage Block ******/ 105/****** Key Storage Block ******/
108#define KSB_START 77 106#define KSB_START 77
109#define KSB_END 127 107#define KSB_END 127
@@ -114,7 +112,7 @@
114#define CCP_JOBID_MASK 0x0000003f 112#define CCP_JOBID_MASK 0x0000003f
115 113
116#define CCP_DMAPOOL_MAX_SIZE 64 114#define CCP_DMAPOOL_MAX_SIZE 64
117#define CCP_DMAPOOL_ALIGN (1 << 5) 115#define CCP_DMAPOOL_ALIGN BIT(5)
118 116
119#define CCP_REVERSE_BUF_SIZE 64 117#define CCP_REVERSE_BUF_SIZE 64
120 118
@@ -142,7 +140,6 @@
142#define CCP_ECC_RESULT_OFFSET 60 140#define CCP_ECC_RESULT_OFFSET 60
143#define CCP_ECC_RESULT_SUCCESS 0x0001 141#define CCP_ECC_RESULT_SUCCESS 0x0001
144 142
145
146struct ccp_device; 143struct ccp_device;
147struct ccp_cmd; 144struct ccp_cmd;
148 145
@@ -261,7 +258,6 @@ struct ccp_device {
261 unsigned int axcache; 258 unsigned int axcache;
262}; 259};
263 260
264
265int ccp_pci_init(void); 261int ccp_pci_init(void);
266void ccp_pci_exit(void); 262void ccp_pci_exit(void);
267 263
diff --git a/drivers/crypto/ccp/ccp-ops.c b/drivers/crypto/ccp/ccp-ops.c
index 8729364261d7..71f2e3c89424 100644
--- a/drivers/crypto/ccp/ccp-ops.c
+++ b/drivers/crypto/ccp/ccp-ops.c
@@ -27,7 +27,6 @@
27 27
28#include "ccp-dev.h" 28#include "ccp-dev.h"
29 29
30
31enum ccp_memtype { 30enum ccp_memtype {
32 CCP_MEMTYPE_SYSTEM = 0, 31 CCP_MEMTYPE_SYSTEM = 0,
33 CCP_MEMTYPE_KSB, 32 CCP_MEMTYPE_KSB,
@@ -515,7 +514,6 @@ static int ccp_init_sg_workarea(struct ccp_sg_workarea *wa, struct device *dev,
515 if (!wa->dma_count) 514 if (!wa->dma_count)
516 return -ENOMEM; 515 return -ENOMEM;
517 516
518
519 return 0; 517 return 0;
520} 518}
521 519
@@ -763,8 +761,9 @@ static void ccp_prepare_data(struct ccp_data *src, struct ccp_data *dst,
763 sg_dst_len = sg_dma_len(dst->sg_wa.sg) - dst->sg_wa.sg_used; 761 sg_dst_len = sg_dma_len(dst->sg_wa.sg) - dst->sg_wa.sg_used;
764 sg_dst_len = min_t(u64, src->sg_wa.bytes_left, sg_dst_len); 762 sg_dst_len = min_t(u64, src->sg_wa.bytes_left, sg_dst_len);
765 op_len = min(sg_src_len, sg_dst_len); 763 op_len = min(sg_src_len, sg_dst_len);
766 } else 764 } else {
767 op_len = sg_src_len; 765 op_len = sg_src_len;
766 }
768 767
769 /* The data operation length will be at least block_size in length 768 /* The data operation length will be at least block_size in length
770 * or the smaller of available sg room remaining for the source or 769 * or the smaller of available sg room remaining for the source or
@@ -1131,9 +1130,9 @@ static int ccp_run_aes_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1131 if (ret) 1130 if (ret)
1132 goto e_ctx; 1131 goto e_ctx;
1133 1132
1134 if (in_place) 1133 if (in_place) {
1135 dst = src; 1134 dst = src;
1136 else { 1135 } else {
1137 ret = ccp_init_data(&dst, cmd_q, aes->dst, aes->src_len, 1136 ret = ccp_init_data(&dst, cmd_q, aes->dst, aes->src_len,
1138 AES_BLOCK_SIZE, DMA_FROM_DEVICE); 1137 AES_BLOCK_SIZE, DMA_FROM_DEVICE);
1139 if (ret) 1138 if (ret)
@@ -1304,9 +1303,9 @@ static int ccp_run_xts_aes_cmd(struct ccp_cmd_queue *cmd_q,
1304 if (ret) 1303 if (ret)
1305 goto e_ctx; 1304 goto e_ctx;
1306 1305
1307 if (in_place) 1306 if (in_place) {
1308 dst = src; 1307 dst = src;
1309 else { 1308 } else {
1310 ret = ccp_init_data(&dst, cmd_q, xts->dst, xts->src_len, 1309 ret = ccp_init_data(&dst, cmd_q, xts->dst, xts->src_len,
1311 unit_size, DMA_FROM_DEVICE); 1310 unit_size, DMA_FROM_DEVICE);
1312 if (ret) 1311 if (ret)
@@ -1451,8 +1450,9 @@ static int ccp_run_sha_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1451 goto e_ctx; 1450 goto e_ctx;
1452 } 1451 }
1453 memcpy(ctx.address, init, CCP_SHA_CTXSIZE); 1452 memcpy(ctx.address, init, CCP_SHA_CTXSIZE);
1454 } else 1453 } else {
1455 ccp_set_dm_area(&ctx, 0, sha->ctx, 0, sha->ctx_len); 1454 ccp_set_dm_area(&ctx, 0, sha->ctx, 0, sha->ctx_len);
1455 }
1456 1456
1457 ret = ccp_copy_to_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx, 1457 ret = ccp_copy_to_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,
1458 CCP_PASSTHRU_BYTESWAP_256BIT); 1458 CCP_PASSTHRU_BYTESWAP_256BIT);
@@ -1732,9 +1732,9 @@ static int ccp_run_passthru_cmd(struct ccp_cmd_queue *cmd_q,
1732 if (ret) 1732 if (ret)
1733 goto e_mask; 1733 goto e_mask;
1734 1734
1735 if (in_place) 1735 if (in_place) {
1736 dst = src; 1736 dst = src;
1737 else { 1737 } else {
1738 ret = ccp_init_data(&dst, cmd_q, pt->dst, pt->src_len, 1738 ret = ccp_init_data(&dst, cmd_q, pt->dst, pt->src_len,
1739 CCP_PASSTHRU_MASKSIZE, DMA_FROM_DEVICE); 1739 CCP_PASSTHRU_MASKSIZE, DMA_FROM_DEVICE);
1740 if (ret) 1740 if (ret)
@@ -1974,7 +1974,7 @@ static int ccp_run_ecc_pm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1974 src.address += CCP_ECC_OPERAND_SIZE; 1974 src.address += CCP_ECC_OPERAND_SIZE;
1975 1975
1976 /* Set the first point Z coordianate to 1 */ 1976 /* Set the first point Z coordianate to 1 */
1977 *(src.address) = 0x01; 1977 *src.address = 0x01;
1978 src.address += CCP_ECC_OPERAND_SIZE; 1978 src.address += CCP_ECC_OPERAND_SIZE;
1979 1979
1980 if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) { 1980 if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) {
@@ -1989,7 +1989,7 @@ static int ccp_run_ecc_pm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1989 src.address += CCP_ECC_OPERAND_SIZE; 1989 src.address += CCP_ECC_OPERAND_SIZE;
1990 1990
1991 /* Set the second point Z coordianate to 1 */ 1991 /* Set the second point Z coordianate to 1 */
1992 *(src.address) = 0x01; 1992 *src.address = 0x01;
1993 src.address += CCP_ECC_OPERAND_SIZE; 1993 src.address += CCP_ECC_OPERAND_SIZE;
1994 } else { 1994 } else {
1995 /* Copy the Domain "a" parameter */ 1995 /* Copy the Domain "a" parameter */
diff --git a/drivers/crypto/ccp/ccp-pci.c b/drivers/crypto/ccp/ccp-pci.c
index 7f89c946adfe..af190d4795a8 100644
--- a/drivers/crypto/ccp/ccp-pci.c
+++ b/drivers/crypto/ccp/ccp-pci.c
@@ -174,11 +174,10 @@ static int ccp_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
174 if (!ccp) 174 if (!ccp)
175 goto e_err; 175 goto e_err;
176 176
177 ccp_pci = kzalloc(sizeof(*ccp_pci), GFP_KERNEL); 177 ccp_pci = devm_kzalloc(dev, sizeof(*ccp_pci), GFP_KERNEL);
178 if (!ccp_pci) { 178 if (!ccp_pci)
179 ret = -ENOMEM; 179 goto e_err;
180 goto e_free1; 180
181 }
182 ccp->dev_specific = ccp_pci; 181 ccp->dev_specific = ccp_pci;
183 ccp->get_irq = ccp_get_irqs; 182 ccp->get_irq = ccp_get_irqs;
184 ccp->free_irq = ccp_free_irqs; 183 ccp->free_irq = ccp_free_irqs;
@@ -186,7 +185,7 @@ static int ccp_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
186 ret = pci_request_regions(pdev, "ccp"); 185 ret = pci_request_regions(pdev, "ccp");
187 if (ret) { 186 if (ret) {
188 dev_err(dev, "pci_request_regions failed (%d)\n", ret); 187 dev_err(dev, "pci_request_regions failed (%d)\n", ret);
189 goto e_free2; 188 goto e_err;
190 } 189 }
191 190
192 ret = pci_enable_device(pdev); 191 ret = pci_enable_device(pdev);
@@ -204,7 +203,7 @@ static int ccp_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
204 203
205 ret = -EIO; 204 ret = -EIO;
206 ccp->io_map = pci_iomap(pdev, bar, 0); 205 ccp->io_map = pci_iomap(pdev, bar, 0);
207 if (ccp->io_map == NULL) { 206 if (!ccp->io_map) {
208 dev_err(dev, "pci_iomap failed\n"); 207 dev_err(dev, "pci_iomap failed\n");
209 goto e_device; 208 goto e_device;
210 } 209 }
@@ -239,12 +238,6 @@ e_device:
239e_regions: 238e_regions:
240 pci_release_regions(pdev); 239 pci_release_regions(pdev);
241 240
242e_free2:
243 kfree(ccp_pci);
244
245e_free1:
246 kfree(ccp);
247
248e_err: 241e_err:
249 dev_notice(dev, "initialization failed\n"); 242 dev_notice(dev, "initialization failed\n");
250 return ret; 243 return ret;
@@ -266,8 +259,6 @@ static void ccp_pci_remove(struct pci_dev *pdev)
266 259
267 pci_release_regions(pdev); 260 pci_release_regions(pdev);
268 261
269 kfree(ccp);
270
271 dev_notice(dev, "disabled\n"); 262 dev_notice(dev, "disabled\n");
272} 263}
273 264
diff --git a/drivers/crypto/ccp/ccp-platform.c b/drivers/crypto/ccp/ccp-platform.c
index 8c50bad25f7e..b1c20b2b5647 100644
--- a/drivers/crypto/ccp/ccp-platform.c
+++ b/drivers/crypto/ccp/ccp-platform.c
@@ -23,9 +23,15 @@
23#include <linux/delay.h> 23#include <linux/delay.h>
24#include <linux/ccp.h> 24#include <linux/ccp.h>
25#include <linux/of.h> 25#include <linux/of.h>
26#include <linux/of_address.h>
27#include <linux/acpi.h>
26 28
27#include "ccp-dev.h" 29#include "ccp-dev.h"
28 30
31struct ccp_platform {
32 int use_acpi;
33 int coherent;
34};
29 35
30static int ccp_get_irq(struct ccp_device *ccp) 36static int ccp_get_irq(struct ccp_device *ccp)
31{ 37{
@@ -84,10 +90,64 @@ static struct resource *ccp_find_mmio_area(struct ccp_device *ccp)
84 return NULL; 90 return NULL;
85} 91}
86 92
93#ifdef CONFIG_ACPI
94static int ccp_acpi_support(struct ccp_device *ccp)
95{
96 struct ccp_platform *ccp_platform = ccp->dev_specific;
97 struct acpi_device *adev = ACPI_COMPANION(ccp->dev);
98 acpi_handle handle;
99 acpi_status status;
100 unsigned long long data;
101 int cca;
102
103 /* Retrieve the device cache coherency value */
104 handle = adev->handle;
105 do {
106 status = acpi_evaluate_integer(handle, "_CCA", NULL, &data);
107 if (!ACPI_FAILURE(status)) {
108 cca = data;
109 break;
110 }
111 } while (!ACPI_FAILURE(status));
112
113 if (ACPI_FAILURE(status)) {
114 dev_err(ccp->dev, "error obtaining acpi coherency value\n");
115 return -EINVAL;
116 }
117
118 ccp_platform->coherent = !!cca;
119
120 return 0;
121}
122#else /* CONFIG_ACPI */
123static int ccp_acpi_support(struct ccp_device *ccp)
124{
125 return -EINVAL;
126}
127#endif
128
129#ifdef CONFIG_OF
130static int ccp_of_support(struct ccp_device *ccp)
131{
132 struct ccp_platform *ccp_platform = ccp->dev_specific;
133
134 ccp_platform->coherent = of_dma_is_coherent(ccp->dev->of_node);
135
136 return 0;
137}
138#else
139static int ccp_of_support(struct ccp_device *ccp)
140{
141 return -EINVAL;
142}
143#endif
144
87static int ccp_platform_probe(struct platform_device *pdev) 145static int ccp_platform_probe(struct platform_device *pdev)
88{ 146{
89 struct ccp_device *ccp; 147 struct ccp_device *ccp;
148 struct ccp_platform *ccp_platform;
90 struct device *dev = &pdev->dev; 149 struct device *dev = &pdev->dev;
150 struct acpi_device *adev = ACPI_COMPANION(dev);
91 struct resource *ior; 151 struct resource *ior;
92 int ret; 152 int ret;
93 153
@@ -96,24 +156,40 @@ static int ccp_platform_probe(struct platform_device *pdev)
96 if (!ccp) 156 if (!ccp)
97 goto e_err; 157 goto e_err;
98 158
99 ccp->dev_specific = NULL; 159 ccp_platform = devm_kzalloc(dev, sizeof(*ccp_platform), GFP_KERNEL);
160 if (!ccp_platform)
161 goto e_err;
162
163 ccp->dev_specific = ccp_platform;
100 ccp->get_irq = ccp_get_irqs; 164 ccp->get_irq = ccp_get_irqs;
101 ccp->free_irq = ccp_free_irqs; 165 ccp->free_irq = ccp_free_irqs;
102 166
167 ccp_platform->use_acpi = (!adev || acpi_disabled) ? 0 : 1;
168
103 ior = ccp_find_mmio_area(ccp); 169 ior = ccp_find_mmio_area(ccp);
104 ccp->io_map = devm_ioremap_resource(dev, ior); 170 ccp->io_map = devm_ioremap_resource(dev, ior);
105 if (IS_ERR(ccp->io_map)) { 171 if (IS_ERR(ccp->io_map)) {
106 ret = PTR_ERR(ccp->io_map); 172 ret = PTR_ERR(ccp->io_map);
107 goto e_free; 173 goto e_err;
108 } 174 }
109 ccp->io_regs = ccp->io_map; 175 ccp->io_regs = ccp->io_map;
110 176
111 if (!dev->dma_mask) 177 if (!dev->dma_mask)
112 dev->dma_mask = &dev->coherent_dma_mask; 178 dev->dma_mask = &dev->coherent_dma_mask;
113 *(dev->dma_mask) = DMA_BIT_MASK(48); 179 ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(48));
114 dev->coherent_dma_mask = DMA_BIT_MASK(48); 180 if (ret) {
181 dev_err(dev, "dma_set_mask_and_coherent failed (%d)\n", ret);
182 goto e_err;
183 }
184
185 if (ccp_platform->use_acpi)
186 ret = ccp_acpi_support(ccp);
187 else
188 ret = ccp_of_support(ccp);
189 if (ret)
190 goto e_err;
115 191
116 if (of_property_read_bool(dev->of_node, "dma-coherent")) 192 if (ccp_platform->coherent)
117 ccp->axcache = CACHE_WB_NO_ALLOC; 193 ccp->axcache = CACHE_WB_NO_ALLOC;
118 else 194 else
119 ccp->axcache = CACHE_NONE; 195 ccp->axcache = CACHE_NONE;
@@ -122,15 +198,12 @@ static int ccp_platform_probe(struct platform_device *pdev)
122 198
123 ret = ccp_init(ccp); 199 ret = ccp_init(ccp);
124 if (ret) 200 if (ret)
125 goto e_free; 201 goto e_err;
126 202
127 dev_notice(dev, "enabled\n"); 203 dev_notice(dev, "enabled\n");
128 204
129 return 0; 205 return 0;
130 206
131e_free:
132 kfree(ccp);
133
134e_err: 207e_err:
135 dev_notice(dev, "initialization failed\n"); 208 dev_notice(dev, "initialization failed\n");
136 return ret; 209 return ret;
@@ -143,8 +216,6 @@ static int ccp_platform_remove(struct platform_device *pdev)
143 216
144 ccp_destroy(ccp); 217 ccp_destroy(ccp);
145 218
146 kfree(ccp);
147
148 dev_notice(dev, "disabled\n"); 219 dev_notice(dev, "disabled\n");
149 220
150 return 0; 221 return 0;
@@ -200,15 +271,29 @@ static int ccp_platform_resume(struct platform_device *pdev)
200} 271}
201#endif 272#endif
202 273
203static const struct of_device_id ccp_platform_ids[] = { 274#ifdef CONFIG_ACPI
275static const struct acpi_device_id ccp_acpi_match[] = {
276 { "AMDI0C00", 0 },
277 { },
278};
279#endif
280
281#ifdef CONFIG_OF
282static const struct of_device_id ccp_of_match[] = {
204 { .compatible = "amd,ccp-seattle-v1a" }, 283 { .compatible = "amd,ccp-seattle-v1a" },
205 { }, 284 { },
206}; 285};
286#endif
207 287
208static struct platform_driver ccp_platform_driver = { 288static struct platform_driver ccp_platform_driver = {
209 .driver = { 289 .driver = {
210 .name = "AMD Cryptographic Coprocessor", 290 .name = "AMD Cryptographic Coprocessor",
211 .of_match_table = ccp_platform_ids, 291#ifdef CONFIG_ACPI
292 .acpi_match_table = ccp_acpi_match,
293#endif
294#ifdef CONFIG_OF
295 .of_match_table = ccp_of_match,
296#endif
212 }, 297 },
213 .probe = ccp_platform_probe, 298 .probe = ccp_platform_probe,
214 .remove = ccp_platform_remove, 299 .remove = ccp_platform_remove,
diff --git a/drivers/crypto/img-hash.c b/drivers/crypto/img-hash.c
new file mode 100644
index 000000000000..ad47d0d61098
--- /dev/null
+++ b/drivers/crypto/img-hash.c
@@ -0,0 +1,1029 @@
1/*
2 * Copyright (c) 2014 Imagination Technologies
3 * Authors: Will Thomas, James Hartley
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License version 2 as published
7 * by the Free Software Foundation.
8 *
9 * Interface structure taken from omap-sham driver
10 */
11
12#include <linux/clk.h>
13#include <linux/dmaengine.h>
14#include <linux/interrupt.h>
15#include <linux/io.h>
16#include <linux/kernel.h>
17#include <linux/module.h>
18#include <linux/of_device.h>
19#include <linux/platform_device.h>
20#include <linux/scatterlist.h>
21
22#include <crypto/internal/hash.h>
23#include <crypto/md5.h>
24#include <crypto/sha.h>
25
26#define CR_RESET 0
27#define CR_RESET_SET 1
28#define CR_RESET_UNSET 0
29
30#define CR_MESSAGE_LENGTH_H 0x4
31#define CR_MESSAGE_LENGTH_L 0x8
32
33#define CR_CONTROL 0xc
34#define CR_CONTROL_BYTE_ORDER_3210 0
35#define CR_CONTROL_BYTE_ORDER_0123 1
36#define CR_CONTROL_BYTE_ORDER_2310 2
37#define CR_CONTROL_BYTE_ORDER_1032 3
38#define CR_CONTROL_BYTE_ORDER_SHIFT 8
39#define CR_CONTROL_ALGO_MD5 0
40#define CR_CONTROL_ALGO_SHA1 1
41#define CR_CONTROL_ALGO_SHA224 2
42#define CR_CONTROL_ALGO_SHA256 3
43
44#define CR_INTSTAT 0x10
45#define CR_INTENAB 0x14
46#define CR_INTCLEAR 0x18
47#define CR_INT_RESULTS_AVAILABLE BIT(0)
48#define CR_INT_NEW_RESULTS_SET BIT(1)
49#define CR_INT_RESULT_READ_ERR BIT(2)
50#define CR_INT_MESSAGE_WRITE_ERROR BIT(3)
51#define CR_INT_STATUS BIT(8)
52
53#define CR_RESULT_QUEUE 0x1c
54#define CR_RSD0 0x40
55#define CR_CORE_REV 0x50
56#define CR_CORE_DES1 0x60
57#define CR_CORE_DES2 0x70
58
59#define DRIVER_FLAGS_BUSY BIT(0)
60#define DRIVER_FLAGS_FINAL BIT(1)
61#define DRIVER_FLAGS_DMA_ACTIVE BIT(2)
62#define DRIVER_FLAGS_OUTPUT_READY BIT(3)
63#define DRIVER_FLAGS_INIT BIT(4)
64#define DRIVER_FLAGS_CPU BIT(5)
65#define DRIVER_FLAGS_DMA_READY BIT(6)
66#define DRIVER_FLAGS_ERROR BIT(7)
67#define DRIVER_FLAGS_SG BIT(8)
68#define DRIVER_FLAGS_SHA1 BIT(18)
69#define DRIVER_FLAGS_SHA224 BIT(19)
70#define DRIVER_FLAGS_SHA256 BIT(20)
71#define DRIVER_FLAGS_MD5 BIT(21)
72
73#define IMG_HASH_QUEUE_LENGTH 20
74#define IMG_HASH_DMA_THRESHOLD 64
75
76#ifdef __LITTLE_ENDIAN
77#define IMG_HASH_BYTE_ORDER CR_CONTROL_BYTE_ORDER_3210
78#else
79#define IMG_HASH_BYTE_ORDER CR_CONTROL_BYTE_ORDER_0123
80#endif
81
82struct img_hash_dev;
83
84struct img_hash_request_ctx {
85 struct img_hash_dev *hdev;
86 u8 digest[SHA256_DIGEST_SIZE] __aligned(sizeof(u32));
87 unsigned long flags;
88 size_t digsize;
89
90 dma_addr_t dma_addr;
91 size_t dma_ct;
92
93 /* sg root */
94 struct scatterlist *sgfirst;
95 /* walk state */
96 struct scatterlist *sg;
97 size_t nents;
98 size_t offset;
99 unsigned int total;
100 size_t sent;
101
102 unsigned long op;
103
104 size_t bufcnt;
105 u8 buffer[0] __aligned(sizeof(u32));
106 struct ahash_request fallback_req;
107};
108
109struct img_hash_ctx {
110 struct img_hash_dev *hdev;
111 unsigned long flags;
112 struct crypto_ahash *fallback;
113};
114
115struct img_hash_dev {
116 struct list_head list;
117 struct device *dev;
118 struct clk *hash_clk;
119 struct clk *sys_clk;
120 void __iomem *io_base;
121
122 phys_addr_t bus_addr;
123 void __iomem *cpu_addr;
124
125 spinlock_t lock;
126 int err;
127 struct tasklet_struct done_task;
128 struct tasklet_struct dma_task;
129
130 unsigned long flags;
131 struct crypto_queue queue;
132 struct ahash_request *req;
133
134 struct dma_chan *dma_lch;
135};
136
137struct img_hash_drv {
138 struct list_head dev_list;
139 spinlock_t lock;
140};
141
142static struct img_hash_drv img_hash = {
143 .dev_list = LIST_HEAD_INIT(img_hash.dev_list),
144 .lock = __SPIN_LOCK_UNLOCKED(img_hash.lock),
145};
146
147static inline u32 img_hash_read(struct img_hash_dev *hdev, u32 offset)
148{
149 return readl_relaxed(hdev->io_base + offset);
150}
151
152static inline void img_hash_write(struct img_hash_dev *hdev,
153 u32 offset, u32 value)
154{
155 writel_relaxed(value, hdev->io_base + offset);
156}
157
158static inline u32 img_hash_read_result_queue(struct img_hash_dev *hdev)
159{
160 return be32_to_cpu(img_hash_read(hdev, CR_RESULT_QUEUE));
161}
162
163static void img_hash_start(struct img_hash_dev *hdev, bool dma)
164{
165 struct img_hash_request_ctx *ctx = ahash_request_ctx(hdev->req);
166 u32 cr = IMG_HASH_BYTE_ORDER << CR_CONTROL_BYTE_ORDER_SHIFT;
167
168 if (ctx->flags & DRIVER_FLAGS_MD5)
169 cr |= CR_CONTROL_ALGO_MD5;
170 else if (ctx->flags & DRIVER_FLAGS_SHA1)
171 cr |= CR_CONTROL_ALGO_SHA1;
172 else if (ctx->flags & DRIVER_FLAGS_SHA224)
173 cr |= CR_CONTROL_ALGO_SHA224;
174 else if (ctx->flags & DRIVER_FLAGS_SHA256)
175 cr |= CR_CONTROL_ALGO_SHA256;
176 dev_dbg(hdev->dev, "Starting hash process\n");
177 img_hash_write(hdev, CR_CONTROL, cr);
178
179 /*
180 * The hardware block requires two cycles between writing the control
181 * register and writing the first word of data in non DMA mode, to
182 * ensure the first data write is not grouped in burst with the control
183 * register write a read is issued to 'flush' the bus.
184 */
185 if (!dma)
186 img_hash_read(hdev, CR_CONTROL);
187}
188
189static int img_hash_xmit_cpu(struct img_hash_dev *hdev, const u8 *buf,
190 size_t length, int final)
191{
192 u32 count, len32;
193 const u32 *buffer = (const u32 *)buf;
194
195 dev_dbg(hdev->dev, "xmit_cpu: length: %zu bytes\n", length);
196
197 if (final)
198 hdev->flags |= DRIVER_FLAGS_FINAL;
199
200 len32 = DIV_ROUND_UP(length, sizeof(u32));
201
202 for (count = 0; count < len32; count++)
203 writel_relaxed(buffer[count], hdev->cpu_addr);
204
205 return -EINPROGRESS;
206}
207
208static void img_hash_dma_callback(void *data)
209{
210 struct img_hash_dev *hdev = (struct img_hash_dev *)data;
211 struct img_hash_request_ctx *ctx = ahash_request_ctx(hdev->req);
212
213 if (ctx->bufcnt) {
214 img_hash_xmit_cpu(hdev, ctx->buffer, ctx->bufcnt, 0);
215 ctx->bufcnt = 0;
216 }
217 if (ctx->sg)
218 tasklet_schedule(&hdev->dma_task);
219}
220
221static int img_hash_xmit_dma(struct img_hash_dev *hdev, struct scatterlist *sg)
222{
223 struct dma_async_tx_descriptor *desc;
224 struct img_hash_request_ctx *ctx = ahash_request_ctx(hdev->req);
225
226 ctx->dma_ct = dma_map_sg(hdev->dev, sg, 1, DMA_MEM_TO_DEV);
227 if (ctx->dma_ct == 0) {
228 dev_err(hdev->dev, "Invalid DMA sg\n");
229 hdev->err = -EINVAL;
230 return -EINVAL;
231 }
232
233 desc = dmaengine_prep_slave_sg(hdev->dma_lch,
234 sg,
235 ctx->dma_ct,
236 DMA_MEM_TO_DEV,
237 DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
238 if (!desc) {
239 dev_err(hdev->dev, "Null DMA descriptor\n");
240 hdev->err = -EINVAL;
241 dma_unmap_sg(hdev->dev, sg, 1, DMA_MEM_TO_DEV);
242 return -EINVAL;
243 }
244 desc->callback = img_hash_dma_callback;
245 desc->callback_param = hdev;
246 dmaengine_submit(desc);
247 dma_async_issue_pending(hdev->dma_lch);
248
249 return 0;
250}
251
252static int img_hash_write_via_cpu(struct img_hash_dev *hdev)
253{
254 struct img_hash_request_ctx *ctx = ahash_request_ctx(hdev->req);
255
256 ctx->bufcnt = sg_copy_to_buffer(hdev->req->src, sg_nents(ctx->sg),
257 ctx->buffer, hdev->req->nbytes);
258
259 ctx->total = hdev->req->nbytes;
260 ctx->bufcnt = 0;
261
262 hdev->flags |= (DRIVER_FLAGS_CPU | DRIVER_FLAGS_FINAL);
263
264 img_hash_start(hdev, false);
265
266 return img_hash_xmit_cpu(hdev, ctx->buffer, ctx->total, 1);
267}
268
269static int img_hash_finish(struct ahash_request *req)
270{
271 struct img_hash_request_ctx *ctx = ahash_request_ctx(req);
272
273 if (!req->result)
274 return -EINVAL;
275
276 memcpy(req->result, ctx->digest, ctx->digsize);
277
278 return 0;
279}
280
281static void img_hash_copy_hash(struct ahash_request *req)
282{
283 struct img_hash_request_ctx *ctx = ahash_request_ctx(req);
284 u32 *hash = (u32 *)ctx->digest;
285 int i;
286
287 for (i = (ctx->digsize / sizeof(u32)) - 1; i >= 0; i--)
288 hash[i] = img_hash_read_result_queue(ctx->hdev);
289}
290
291static void img_hash_finish_req(struct ahash_request *req, int err)
292{
293 struct img_hash_request_ctx *ctx = ahash_request_ctx(req);
294 struct img_hash_dev *hdev = ctx->hdev;
295
296 if (!err) {
297 img_hash_copy_hash(req);
298 if (DRIVER_FLAGS_FINAL & hdev->flags)
299 err = img_hash_finish(req);
300 } else {
301 dev_warn(hdev->dev, "Hash failed with error %d\n", err);
302 ctx->flags |= DRIVER_FLAGS_ERROR;
303 }
304
305 hdev->flags &= ~(DRIVER_FLAGS_DMA_READY | DRIVER_FLAGS_OUTPUT_READY |
306 DRIVER_FLAGS_CPU | DRIVER_FLAGS_BUSY | DRIVER_FLAGS_FINAL);
307
308 if (req->base.complete)
309 req->base.complete(&req->base, err);
310}
311
312static int img_hash_write_via_dma(struct img_hash_dev *hdev)
313{
314 struct img_hash_request_ctx *ctx = ahash_request_ctx(hdev->req);
315
316 img_hash_start(hdev, true);
317
318 dev_dbg(hdev->dev, "xmit dma size: %d\n", ctx->total);
319
320 if (!ctx->total)
321 hdev->flags |= DRIVER_FLAGS_FINAL;
322
323 hdev->flags |= DRIVER_FLAGS_DMA_ACTIVE | DRIVER_FLAGS_FINAL;
324
325 tasklet_schedule(&hdev->dma_task);
326
327 return -EINPROGRESS;
328}
329
330static int img_hash_dma_init(struct img_hash_dev *hdev)
331{
332 struct dma_slave_config dma_conf;
333 int err = -EINVAL;
334
335 hdev->dma_lch = dma_request_slave_channel(hdev->dev, "tx");
336 if (!hdev->dma_lch) {
337 dev_err(hdev->dev, "Couldn't aquire a slave DMA channel.\n");
338 return -EBUSY;
339 }
340 dma_conf.direction = DMA_MEM_TO_DEV;
341 dma_conf.dst_addr = hdev->bus_addr;
342 dma_conf.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
343 dma_conf.dst_maxburst = 16;
344 dma_conf.device_fc = false;
345
346 err = dmaengine_slave_config(hdev->dma_lch, &dma_conf);
347 if (err) {
348 dev_err(hdev->dev, "Couldn't configure DMA slave.\n");
349 dma_release_channel(hdev->dma_lch);
350 return err;
351 }
352
353 return 0;
354}
355
356static void img_hash_dma_task(unsigned long d)
357{
358 struct img_hash_dev *hdev = (struct img_hash_dev *)d;
359 struct img_hash_request_ctx *ctx = ahash_request_ctx(hdev->req);
360 u8 *addr;
361 size_t nbytes, bleft, wsend, len, tbc;
362 struct scatterlist tsg;
363
364 if (!ctx->sg)
365 return;
366
367 addr = sg_virt(ctx->sg);
368 nbytes = ctx->sg->length - ctx->offset;
369
370 /*
371 * The hash accelerator does not support a data valid mask. This means
372 * that if each dma (i.e. per page) is not a multiple of 4 bytes, the
373 * padding bytes in the last word written by that dma would erroneously
374 * be included in the hash. To avoid this we round down the transfer,
375 * and add the excess to the start of the next dma. It does not matter
376 * that the final dma may not be a multiple of 4 bytes as the hashing
377 * block is programmed to accept the correct number of bytes.
378 */
379
380 bleft = nbytes % 4;
381 wsend = (nbytes / 4);
382
383 if (wsend) {
384 sg_init_one(&tsg, addr + ctx->offset, wsend * 4);
385 if (img_hash_xmit_dma(hdev, &tsg)) {
386 dev_err(hdev->dev, "DMA failed, falling back to CPU");
387 ctx->flags |= DRIVER_FLAGS_CPU;
388 hdev->err = 0;
389 img_hash_xmit_cpu(hdev, addr + ctx->offset,
390 wsend * 4, 0);
391 ctx->sent += wsend * 4;
392 wsend = 0;
393 } else {
394 ctx->sent += wsend * 4;
395 }
396 }
397
398 if (bleft) {
399 ctx->bufcnt = sg_pcopy_to_buffer(ctx->sgfirst, ctx->nents,
400 ctx->buffer, bleft, ctx->sent);
401 tbc = 0;
402 ctx->sg = sg_next(ctx->sg);
403 while (ctx->sg && (ctx->bufcnt < 4)) {
404 len = ctx->sg->length;
405 if (likely(len > (4 - ctx->bufcnt)))
406 len = 4 - ctx->bufcnt;
407 tbc = sg_pcopy_to_buffer(ctx->sgfirst, ctx->nents,
408 ctx->buffer + ctx->bufcnt, len,
409 ctx->sent + ctx->bufcnt);
410 ctx->bufcnt += tbc;
411 if (tbc >= ctx->sg->length) {
412 ctx->sg = sg_next(ctx->sg);
413 tbc = 0;
414 }
415 }
416
417 ctx->sent += ctx->bufcnt;
418 ctx->offset = tbc;
419
420 if (!wsend)
421 img_hash_dma_callback(hdev);
422 } else {
423 ctx->offset = 0;
424 ctx->sg = sg_next(ctx->sg);
425 }
426}
427
428static int img_hash_write_via_dma_stop(struct img_hash_dev *hdev)
429{
430 struct img_hash_request_ctx *ctx = ahash_request_ctx(hdev->req);
431
432 if (ctx->flags & DRIVER_FLAGS_SG)
433 dma_unmap_sg(hdev->dev, ctx->sg, ctx->dma_ct, DMA_TO_DEVICE);
434
435 return 0;
436}
437
438static int img_hash_process_data(struct img_hash_dev *hdev)
439{
440 struct ahash_request *req = hdev->req;
441 struct img_hash_request_ctx *ctx = ahash_request_ctx(req);
442 int err = 0;
443
444 ctx->bufcnt = 0;
445
446 if (req->nbytes >= IMG_HASH_DMA_THRESHOLD) {
447 dev_dbg(hdev->dev, "process data request(%d bytes) using DMA\n",
448 req->nbytes);
449 err = img_hash_write_via_dma(hdev);
450 } else {
451 dev_dbg(hdev->dev, "process data request(%d bytes) using CPU\n",
452 req->nbytes);
453 err = img_hash_write_via_cpu(hdev);
454 }
455 return err;
456}
457
458static int img_hash_hw_init(struct img_hash_dev *hdev)
459{
460 unsigned long long nbits;
461 u32 u, l;
462
463 img_hash_write(hdev, CR_RESET, CR_RESET_SET);
464 img_hash_write(hdev, CR_RESET, CR_RESET_UNSET);
465 img_hash_write(hdev, CR_INTENAB, CR_INT_NEW_RESULTS_SET);
466
467 nbits = (u64)hdev->req->nbytes << 3;
468 u = nbits >> 32;
469 l = nbits;
470 img_hash_write(hdev, CR_MESSAGE_LENGTH_H, u);
471 img_hash_write(hdev, CR_MESSAGE_LENGTH_L, l);
472
473 if (!(DRIVER_FLAGS_INIT & hdev->flags)) {
474 hdev->flags |= DRIVER_FLAGS_INIT;
475 hdev->err = 0;
476 }
477 dev_dbg(hdev->dev, "hw initialized, nbits: %llx\n", nbits);
478 return 0;
479}
480
481static int img_hash_init(struct ahash_request *req)
482{
483 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
484 struct img_hash_request_ctx *rctx = ahash_request_ctx(req);
485 struct img_hash_ctx *ctx = crypto_ahash_ctx(tfm);
486
487 ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback);
488 rctx->fallback_req.base.flags = req->base.flags
489 & CRYPTO_TFM_REQ_MAY_SLEEP;
490
491 return crypto_ahash_init(&rctx->fallback_req);
492}
493
494static int img_hash_handle_queue(struct img_hash_dev *hdev,
495 struct ahash_request *req)
496{
497 struct crypto_async_request *async_req, *backlog;
498 struct img_hash_request_ctx *ctx;
499 unsigned long flags;
500 int err = 0, res = 0;
501
502 spin_lock_irqsave(&hdev->lock, flags);
503
504 if (req)
505 res = ahash_enqueue_request(&hdev->queue, req);
506
507 if (DRIVER_FLAGS_BUSY & hdev->flags) {
508 spin_unlock_irqrestore(&hdev->lock, flags);
509 return res;
510 }
511
512 backlog = crypto_get_backlog(&hdev->queue);
513 async_req = crypto_dequeue_request(&hdev->queue);
514 if (async_req)
515 hdev->flags |= DRIVER_FLAGS_BUSY;
516
517 spin_unlock_irqrestore(&hdev->lock, flags);
518
519 if (!async_req)
520 return res;
521
522 if (backlog)
523 backlog->complete(backlog, -EINPROGRESS);
524
525 req = ahash_request_cast(async_req);
526 hdev->req = req;
527
528 ctx = ahash_request_ctx(req);
529
530 dev_info(hdev->dev, "processing req, op: %lu, bytes: %d\n",
531 ctx->op, req->nbytes);
532
533 err = img_hash_hw_init(hdev);
534
535 if (!err)
536 err = img_hash_process_data(hdev);
537
538 if (err != -EINPROGRESS) {
539 /* done_task will not finish so do it here */
540 img_hash_finish_req(req, err);
541 }
542 return res;
543}
544
545static int img_hash_update(struct ahash_request *req)
546{
547 struct img_hash_request_ctx *rctx = ahash_request_ctx(req);
548 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
549 struct img_hash_ctx *ctx = crypto_ahash_ctx(tfm);
550
551 ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback);
552 rctx->fallback_req.base.flags = req->base.flags
553 & CRYPTO_TFM_REQ_MAY_SLEEP;
554 rctx->fallback_req.nbytes = req->nbytes;
555 rctx->fallback_req.src = req->src;
556
557 return crypto_ahash_update(&rctx->fallback_req);
558}
559
560static int img_hash_final(struct ahash_request *req)
561{
562 struct img_hash_request_ctx *rctx = ahash_request_ctx(req);
563 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
564 struct img_hash_ctx *ctx = crypto_ahash_ctx(tfm);
565
566 ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback);
567 rctx->fallback_req.base.flags = req->base.flags
568 & CRYPTO_TFM_REQ_MAY_SLEEP;
569 rctx->fallback_req.result = req->result;
570
571 return crypto_ahash_final(&rctx->fallback_req);
572}
573
574static int img_hash_finup(struct ahash_request *req)
575{
576 struct img_hash_request_ctx *rctx = ahash_request_ctx(req);
577 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
578 struct img_hash_ctx *ctx = crypto_ahash_ctx(tfm);
579
580 ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback);
581 rctx->fallback_req.base.flags = req->base.flags
582 & CRYPTO_TFM_REQ_MAY_SLEEP;
583 rctx->fallback_req.nbytes = req->nbytes;
584 rctx->fallback_req.src = req->src;
585 rctx->fallback_req.result = req->result;
586
587 return crypto_ahash_finup(&rctx->fallback_req);
588}
589
590static int img_hash_digest(struct ahash_request *req)
591{
592 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
593 struct img_hash_ctx *tctx = crypto_ahash_ctx(tfm);
594 struct img_hash_request_ctx *ctx = ahash_request_ctx(req);
595 struct img_hash_dev *hdev = NULL;
596 struct img_hash_dev *tmp;
597 int err;
598
599 spin_lock(&img_hash.lock);
600 if (!tctx->hdev) {
601 list_for_each_entry(tmp, &img_hash.dev_list, list) {
602 hdev = tmp;
603 break;
604 }
605 tctx->hdev = hdev;
606
607 } else {
608 hdev = tctx->hdev;
609 }
610
611 spin_unlock(&img_hash.lock);
612 ctx->hdev = hdev;
613 ctx->flags = 0;
614 ctx->digsize = crypto_ahash_digestsize(tfm);
615
616 switch (ctx->digsize) {
617 case SHA1_DIGEST_SIZE:
618 ctx->flags |= DRIVER_FLAGS_SHA1;
619 break;
620 case SHA256_DIGEST_SIZE:
621 ctx->flags |= DRIVER_FLAGS_SHA256;
622 break;
623 case SHA224_DIGEST_SIZE:
624 ctx->flags |= DRIVER_FLAGS_SHA224;
625 break;
626 case MD5_DIGEST_SIZE:
627 ctx->flags |= DRIVER_FLAGS_MD5;
628 break;
629 default:
630 return -EINVAL;
631 }
632
633 ctx->bufcnt = 0;
634 ctx->offset = 0;
635 ctx->sent = 0;
636 ctx->total = req->nbytes;
637 ctx->sg = req->src;
638 ctx->sgfirst = req->src;
639 ctx->nents = sg_nents(ctx->sg);
640
641 err = img_hash_handle_queue(tctx->hdev, req);
642
643 return err;
644}
645
646static int img_hash_cra_init(struct crypto_tfm *tfm)
647{
648 struct img_hash_ctx *ctx = crypto_tfm_ctx(tfm);
649 const char *alg_name = crypto_tfm_alg_name(tfm);
650 int err = -ENOMEM;
651
652 ctx->fallback = crypto_alloc_ahash(alg_name, 0,
653 CRYPTO_ALG_NEED_FALLBACK);
654 if (IS_ERR(ctx->fallback)) {
655 pr_err("img_hash: Could not load fallback driver.\n");
656 err = PTR_ERR(ctx->fallback);
657 goto err;
658 }
659 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
660 sizeof(struct img_hash_request_ctx) +
661 IMG_HASH_DMA_THRESHOLD);
662
663 return 0;
664
665err:
666 return err;
667}
668
669static void img_hash_cra_exit(struct crypto_tfm *tfm)
670{
671 struct img_hash_ctx *tctx = crypto_tfm_ctx(tfm);
672
673 crypto_free_ahash(tctx->fallback);
674}
675
676static irqreturn_t img_irq_handler(int irq, void *dev_id)
677{
678 struct img_hash_dev *hdev = dev_id;
679 u32 reg;
680
681 reg = img_hash_read(hdev, CR_INTSTAT);
682 img_hash_write(hdev, CR_INTCLEAR, reg);
683
684 if (reg & CR_INT_NEW_RESULTS_SET) {
685 dev_dbg(hdev->dev, "IRQ CR_INT_NEW_RESULTS_SET\n");
686 if (DRIVER_FLAGS_BUSY & hdev->flags) {
687 hdev->flags |= DRIVER_FLAGS_OUTPUT_READY;
688 if (!(DRIVER_FLAGS_CPU & hdev->flags))
689 hdev->flags |= DRIVER_FLAGS_DMA_READY;
690 tasklet_schedule(&hdev->done_task);
691 } else {
692 dev_warn(hdev->dev,
693 "HASH interrupt when no active requests.\n");
694 }
695 } else if (reg & CR_INT_RESULTS_AVAILABLE) {
696 dev_warn(hdev->dev,
697 "IRQ triggered before the hash had completed\n");
698 } else if (reg & CR_INT_RESULT_READ_ERR) {
699 dev_warn(hdev->dev,
700 "Attempt to read from an empty result queue\n");
701 } else if (reg & CR_INT_MESSAGE_WRITE_ERROR) {
702 dev_warn(hdev->dev,
703 "Data written before the hardware was configured\n");
704 }
705 return IRQ_HANDLED;
706}
707
708static struct ahash_alg img_algs[] = {
709 {
710 .init = img_hash_init,
711 .update = img_hash_update,
712 .final = img_hash_final,
713 .finup = img_hash_finup,
714 .digest = img_hash_digest,
715 .halg = {
716 .digestsize = MD5_DIGEST_SIZE,
717 .base = {
718 .cra_name = "md5",
719 .cra_driver_name = "img-md5",
720 .cra_priority = 300,
721 .cra_flags =
722 CRYPTO_ALG_ASYNC |
723 CRYPTO_ALG_NEED_FALLBACK,
724 .cra_blocksize = MD5_HMAC_BLOCK_SIZE,
725 .cra_ctxsize = sizeof(struct img_hash_ctx),
726 .cra_init = img_hash_cra_init,
727 .cra_exit = img_hash_cra_exit,
728 .cra_module = THIS_MODULE,
729 }
730 }
731 },
732 {
733 .init = img_hash_init,
734 .update = img_hash_update,
735 .final = img_hash_final,
736 .finup = img_hash_finup,
737 .digest = img_hash_digest,
738 .halg = {
739 .digestsize = SHA1_DIGEST_SIZE,
740 .base = {
741 .cra_name = "sha1",
742 .cra_driver_name = "img-sha1",
743 .cra_priority = 300,
744 .cra_flags =
745 CRYPTO_ALG_ASYNC |
746 CRYPTO_ALG_NEED_FALLBACK,
747 .cra_blocksize = SHA1_BLOCK_SIZE,
748 .cra_ctxsize = sizeof(struct img_hash_ctx),
749 .cra_init = img_hash_cra_init,
750 .cra_exit = img_hash_cra_exit,
751 .cra_module = THIS_MODULE,
752 }
753 }
754 },
755 {
756 .init = img_hash_init,
757 .update = img_hash_update,
758 .final = img_hash_final,
759 .finup = img_hash_finup,
760 .digest = img_hash_digest,
761 .halg = {
762 .digestsize = SHA224_DIGEST_SIZE,
763 .base = {
764 .cra_name = "sha224",
765 .cra_driver_name = "img-sha224",
766 .cra_priority = 300,
767 .cra_flags =
768 CRYPTO_ALG_ASYNC |
769 CRYPTO_ALG_NEED_FALLBACK,
770 .cra_blocksize = SHA224_BLOCK_SIZE,
771 .cra_ctxsize = sizeof(struct img_hash_ctx),
772 .cra_init = img_hash_cra_init,
773 .cra_exit = img_hash_cra_exit,
774 .cra_module = THIS_MODULE,
775 }
776 }
777 },
778 {
779 .init = img_hash_init,
780 .update = img_hash_update,
781 .final = img_hash_final,
782 .finup = img_hash_finup,
783 .digest = img_hash_digest,
784 .halg = {
785 .digestsize = SHA256_DIGEST_SIZE,
786 .base = {
787 .cra_name = "sha256",
788 .cra_driver_name = "img-sha256",
789 .cra_priority = 300,
790 .cra_flags =
791 CRYPTO_ALG_ASYNC |
792 CRYPTO_ALG_NEED_FALLBACK,
793 .cra_blocksize = SHA256_BLOCK_SIZE,
794 .cra_ctxsize = sizeof(struct img_hash_ctx),
795 .cra_init = img_hash_cra_init,
796 .cra_exit = img_hash_cra_exit,
797 .cra_module = THIS_MODULE,
798 }
799 }
800 }
801};
802
803static int img_register_algs(struct img_hash_dev *hdev)
804{
805 int i, err;
806
807 for (i = 0; i < ARRAY_SIZE(img_algs); i++) {
808 err = crypto_register_ahash(&img_algs[i]);
809 if (err)
810 goto err_reg;
811 }
812 return 0;
813
814err_reg:
815 for (; i--; )
816 crypto_unregister_ahash(&img_algs[i]);
817
818 return err;
819}
820
821static int img_unregister_algs(struct img_hash_dev *hdev)
822{
823 int i;
824
825 for (i = 0; i < ARRAY_SIZE(img_algs); i++)
826 crypto_unregister_ahash(&img_algs[i]);
827 return 0;
828}
829
830static void img_hash_done_task(unsigned long data)
831{
832 struct img_hash_dev *hdev = (struct img_hash_dev *)data;
833 int err = 0;
834
835 if (hdev->err == -EINVAL) {
836 err = hdev->err;
837 goto finish;
838 }
839
840 if (!(DRIVER_FLAGS_BUSY & hdev->flags)) {
841 img_hash_handle_queue(hdev, NULL);
842 return;
843 }
844
845 if (DRIVER_FLAGS_CPU & hdev->flags) {
846 if (DRIVER_FLAGS_OUTPUT_READY & hdev->flags) {
847 hdev->flags &= ~DRIVER_FLAGS_OUTPUT_READY;
848 goto finish;
849 }
850 } else if (DRIVER_FLAGS_DMA_READY & hdev->flags) {
851 if (DRIVER_FLAGS_DMA_ACTIVE & hdev->flags) {
852 hdev->flags &= ~DRIVER_FLAGS_DMA_ACTIVE;
853 img_hash_write_via_dma_stop(hdev);
854 if (hdev->err) {
855 err = hdev->err;
856 goto finish;
857 }
858 }
859 if (DRIVER_FLAGS_OUTPUT_READY & hdev->flags) {
860 hdev->flags &= ~(DRIVER_FLAGS_DMA_READY |
861 DRIVER_FLAGS_OUTPUT_READY);
862 goto finish;
863 }
864 }
865 return;
866
867finish:
868 img_hash_finish_req(hdev->req, err);
869}
870
871static const struct of_device_id img_hash_match[] = {
872 { .compatible = "img,hash-accelerator" },
873 {}
874};
875MODULE_DEVICE_TABLE(of, img_hash_match);
876
877static int img_hash_probe(struct platform_device *pdev)
878{
879 struct img_hash_dev *hdev;
880 struct device *dev = &pdev->dev;
881 struct resource *hash_res;
882 int irq;
883 int err;
884
885 hdev = devm_kzalloc(dev, sizeof(*hdev), GFP_KERNEL);
886 if (hdev == NULL)
887 return -ENOMEM;
888
889 spin_lock_init(&hdev->lock);
890
891 hdev->dev = dev;
892
893 platform_set_drvdata(pdev, hdev);
894
895 INIT_LIST_HEAD(&hdev->list);
896
897 tasklet_init(&hdev->done_task, img_hash_done_task, (unsigned long)hdev);
898 tasklet_init(&hdev->dma_task, img_hash_dma_task, (unsigned long)hdev);
899
900 crypto_init_queue(&hdev->queue, IMG_HASH_QUEUE_LENGTH);
901
902 /* Register bank */
903 hash_res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
904
905 hdev->io_base = devm_ioremap_resource(dev, hash_res);
906 if (IS_ERR(hdev->io_base)) {
907 err = PTR_ERR(hdev->io_base);
908 dev_err(dev, "can't ioremap, returned %d\n", err);
909
910 goto res_err;
911 }
912
913 /* Write port (DMA or CPU) */
914 hash_res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
915 hdev->cpu_addr = devm_ioremap_resource(dev, hash_res);
916 if (IS_ERR(hdev->cpu_addr)) {
917 dev_err(dev, "can't ioremap write port\n");
918 err = PTR_ERR(hdev->cpu_addr);
919 goto res_err;
920 }
921 hdev->bus_addr = hash_res->start;
922
923 irq = platform_get_irq(pdev, 0);
924 if (irq < 0) {
925 dev_err(dev, "no IRQ resource info\n");
926 err = irq;
927 goto res_err;
928 }
929
930 err = devm_request_irq(dev, irq, img_irq_handler, 0,
931 dev_name(dev), hdev);
932 if (err) {
933 dev_err(dev, "unable to request irq\n");
934 goto res_err;
935 }
936 dev_dbg(dev, "using IRQ channel %d\n", irq);
937
938 hdev->hash_clk = devm_clk_get(&pdev->dev, "hash");
939 if (IS_ERR(hdev->hash_clk)) {
940 dev_err(dev, "clock initialization failed.\n");
941 err = PTR_ERR(hdev->hash_clk);
942 goto res_err;
943 }
944
945 hdev->sys_clk = devm_clk_get(&pdev->dev, "sys");
946 if (IS_ERR(hdev->sys_clk)) {
947 dev_err(dev, "clock initialization failed.\n");
948 err = PTR_ERR(hdev->sys_clk);
949 goto res_err;
950 }
951
952 err = clk_prepare_enable(hdev->hash_clk);
953 if (err)
954 goto res_err;
955
956 err = clk_prepare_enable(hdev->sys_clk);
957 if (err)
958 goto clk_err;
959
960 err = img_hash_dma_init(hdev);
961 if (err)
962 goto dma_err;
963
964 dev_dbg(dev, "using %s for DMA transfers\n",
965 dma_chan_name(hdev->dma_lch));
966
967 spin_lock(&img_hash.lock);
968 list_add_tail(&hdev->list, &img_hash.dev_list);
969 spin_unlock(&img_hash.lock);
970
971 err = img_register_algs(hdev);
972 if (err)
973 goto err_algs;
974 dev_dbg(dev, "Img MD5/SHA1/SHA224/SHA256 Hardware accelerator initialized\n");
975
976 return 0;
977
978err_algs:
979 spin_lock(&img_hash.lock);
980 list_del(&hdev->list);
981 spin_unlock(&img_hash.lock);
982 dma_release_channel(hdev->dma_lch);
983dma_err:
984 clk_disable_unprepare(hdev->sys_clk);
985clk_err:
986 clk_disable_unprepare(hdev->hash_clk);
987res_err:
988 tasklet_kill(&hdev->done_task);
989 tasklet_kill(&hdev->dma_task);
990
991 return err;
992}
993
994static int img_hash_remove(struct platform_device *pdev)
995{
996 static struct img_hash_dev *hdev;
997
998 hdev = platform_get_drvdata(pdev);
999 spin_lock(&img_hash.lock);
1000 list_del(&hdev->list);
1001 spin_unlock(&img_hash.lock);
1002
1003 img_unregister_algs(hdev);
1004
1005 tasklet_kill(&hdev->done_task);
1006 tasklet_kill(&hdev->dma_task);
1007
1008 dma_release_channel(hdev->dma_lch);
1009
1010 clk_disable_unprepare(hdev->hash_clk);
1011 clk_disable_unprepare(hdev->sys_clk);
1012
1013 return 0;
1014}
1015
1016static struct platform_driver img_hash_driver = {
1017 .probe = img_hash_probe,
1018 .remove = img_hash_remove,
1019 .driver = {
1020 .name = "img-hash-accelerator",
1021 .of_match_table = of_match_ptr(img_hash_match),
1022 }
1023};
1024module_platform_driver(img_hash_driver);
1025
1026MODULE_LICENSE("GPL v2");
1027MODULE_DESCRIPTION("Imgtec SHA1/224/256 & MD5 hw accelerator driver");
1028MODULE_AUTHOR("Will Thomas.");
1029MODULE_AUTHOR("James Hartley <james.hartley@imgtec.com>");
diff --git a/drivers/crypto/mxs-dcp.c b/drivers/crypto/mxs-dcp.c
index 829d6394fb33..59ed54e464a9 100644
--- a/drivers/crypto/mxs-dcp.c
+++ b/drivers/crypto/mxs-dcp.c
@@ -153,7 +153,7 @@ static int mxs_dcp_start_dma(struct dcp_async_ctx *actx)
153 struct dcp *sdcp = global_sdcp; 153 struct dcp *sdcp = global_sdcp;
154 const int chan = actx->chan; 154 const int chan = actx->chan;
155 uint32_t stat; 155 uint32_t stat;
156 int ret; 156 unsigned long ret;
157 struct dcp_dma_desc *desc = &sdcp->coh->desc[actx->chan]; 157 struct dcp_dma_desc *desc = &sdcp->coh->desc[actx->chan];
158 158
159 dma_addr_t desc_phys = dma_map_single(sdcp->dev, desc, sizeof(*desc), 159 dma_addr_t desc_phys = dma_map_single(sdcp->dev, desc, sizeof(*desc),
diff --git a/drivers/crypto/omap-aes.c b/drivers/crypto/omap-aes.c
index 42f95a4326b0..9a28b7e07c71 100644
--- a/drivers/crypto/omap-aes.c
+++ b/drivers/crypto/omap-aes.c
@@ -554,15 +554,23 @@ static int omap_aes_crypt_dma_stop(struct omap_aes_dev *dd)
554 return err; 554 return err;
555} 555}
556 556
557static int omap_aes_check_aligned(struct scatterlist *sg) 557static int omap_aes_check_aligned(struct scatterlist *sg, int total)
558{ 558{
559 int len = 0;
560
559 while (sg) { 561 while (sg) {
560 if (!IS_ALIGNED(sg->offset, 4)) 562 if (!IS_ALIGNED(sg->offset, 4))
561 return -1; 563 return -1;
562 if (!IS_ALIGNED(sg->length, AES_BLOCK_SIZE)) 564 if (!IS_ALIGNED(sg->length, AES_BLOCK_SIZE))
563 return -1; 565 return -1;
566
567 len += sg->length;
564 sg = sg_next(sg); 568 sg = sg_next(sg);
565 } 569 }
570
571 if (len != total)
572 return -1;
573
566 return 0; 574 return 0;
567} 575}
568 576
@@ -633,8 +641,8 @@ static int omap_aes_handle_queue(struct omap_aes_dev *dd,
633 dd->in_sg = req->src; 641 dd->in_sg = req->src;
634 dd->out_sg = req->dst; 642 dd->out_sg = req->dst;
635 643
636 if (omap_aes_check_aligned(dd->in_sg) || 644 if (omap_aes_check_aligned(dd->in_sg, dd->total) ||
637 omap_aes_check_aligned(dd->out_sg)) { 645 omap_aes_check_aligned(dd->out_sg, dd->total)) {
638 if (omap_aes_copy_sgs(dd)) 646 if (omap_aes_copy_sgs(dd))
639 pr_err("Failed to copy SGs for unaligned cases\n"); 647 pr_err("Failed to copy SGs for unaligned cases\n");
640 dd->sgs_copied = 1; 648 dd->sgs_copied = 1;
diff --git a/drivers/crypto/omap-sham.c b/drivers/crypto/omap-sham.c
index 3c76696ee578..4d63e0d4da9a 100644
--- a/drivers/crypto/omap-sham.c
+++ b/drivers/crypto/omap-sham.c
@@ -640,6 +640,7 @@ static size_t omap_sham_append_sg(struct omap_sham_reqctx *ctx)
640 640
641 while (ctx->sg) { 641 while (ctx->sg) {
642 vaddr = kmap_atomic(sg_page(ctx->sg)); 642 vaddr = kmap_atomic(sg_page(ctx->sg));
643 vaddr += ctx->sg->offset;
643 644
644 count = omap_sham_append_buffer(ctx, 645 count = omap_sham_append_buffer(ctx,
645 vaddr + ctx->offset, 646 vaddr + ctx->offset,
@@ -1945,6 +1946,7 @@ static int omap_sham_probe(struct platform_device *pdev)
1945 dd->flags |= dd->pdata->flags; 1946 dd->flags |= dd->pdata->flags;
1946 1947
1947 pm_runtime_enable(dev); 1948 pm_runtime_enable(dev);
1949 pm_runtime_irq_safe(dev);
1948 pm_runtime_get_sync(dev); 1950 pm_runtime_get_sync(dev);
1949 rev = omap_sham_read(dd, SHA_REG_REV(dd)); 1951 rev = omap_sham_read(dd, SHA_REG_REV(dd));
1950 pm_runtime_put_sync(&pdev->dev); 1952 pm_runtime_put_sync(&pdev->dev);
diff --git a/drivers/crypto/qat/qat_common/adf_accel_devices.h b/drivers/crypto/qat/qat_common/adf_accel_devices.h
index 19c0efa29ab3..f22ce7169fa5 100644
--- a/drivers/crypto/qat/qat_common/adf_accel_devices.h
+++ b/drivers/crypto/qat/qat_common/adf_accel_devices.h
@@ -52,7 +52,6 @@
52#include <linux/io.h> 52#include <linux/io.h>
53#include "adf_cfg_common.h" 53#include "adf_cfg_common.h"
54 54
55#define PCI_VENDOR_ID_INTEL 0x8086
56#define ADF_DH895XCC_DEVICE_NAME "dh895xcc" 55#define ADF_DH895XCC_DEVICE_NAME "dh895xcc"
57#define ADF_DH895XCC_PCI_DEVICE_ID 0x435 56#define ADF_DH895XCC_PCI_DEVICE_ID 0x435
58#define ADF_PCI_MAX_BARS 3 57#define ADF_PCI_MAX_BARS 3
diff --git a/drivers/crypto/qat/qat_common/adf_accel_engine.c b/drivers/crypto/qat/qat_common/adf_accel_engine.c
index c77453b900a3..7f8b66c915ed 100644
--- a/drivers/crypto/qat/qat_common/adf_accel_engine.c
+++ b/drivers/crypto/qat/qat_common/adf_accel_engine.c
@@ -60,36 +60,40 @@ int adf_ae_fw_load(struct adf_accel_dev *accel_dev)
60 60
61 if (request_firmware(&loader_data->uof_fw, hw_device->fw_name, 61 if (request_firmware(&loader_data->uof_fw, hw_device->fw_name,
62 &accel_dev->accel_pci_dev.pci_dev->dev)) { 62 &accel_dev->accel_pci_dev.pci_dev->dev)) {
63 pr_err("QAT: Failed to load firmware %s\n", hw_device->fw_name); 63 dev_err(&GET_DEV(accel_dev), "Failed to load firmware %s\n",
64 hw_device->fw_name);
64 return -EFAULT; 65 return -EFAULT;
65 } 66 }
66 67
67 uof_size = loader_data->uof_fw->size; 68 uof_size = loader_data->uof_fw->size;
68 uof_addr = (void *)loader_data->uof_fw->data; 69 uof_addr = (void *)loader_data->uof_fw->data;
69 if (qat_uclo_map_uof_obj(loader_data->fw_loader, uof_addr, uof_size)) { 70 if (qat_uclo_map_uof_obj(loader_data->fw_loader, uof_addr, uof_size)) {
70 pr_err("QAT: Failed to map UOF\n"); 71 dev_err(&GET_DEV(accel_dev), "Failed to map UOF\n");
71 goto out_err; 72 goto out_err;
72 } 73 }
73 if (qat_uclo_wr_all_uimage(loader_data->fw_loader)) { 74 if (qat_uclo_wr_all_uimage(loader_data->fw_loader)) {
74 pr_err("QAT: Failed to map UOF\n"); 75 dev_err(&GET_DEV(accel_dev), "Failed to map UOF\n");
75 goto out_err; 76 goto out_err;
76 } 77 }
77 return 0; 78 return 0;
78 79
79out_err: 80out_err:
80 release_firmware(loader_data->uof_fw); 81 adf_ae_fw_release(accel_dev);
81 return -EFAULT; 82 return -EFAULT;
82} 83}
83 84
84int adf_ae_fw_release(struct adf_accel_dev *accel_dev) 85void adf_ae_fw_release(struct adf_accel_dev *accel_dev)
85{ 86{
86 struct adf_fw_loader_data *loader_data = accel_dev->fw_loader; 87 struct adf_fw_loader_data *loader_data = accel_dev->fw_loader;
87 88
88 release_firmware(loader_data->uof_fw);
89 qat_uclo_del_uof_obj(loader_data->fw_loader); 89 qat_uclo_del_uof_obj(loader_data->fw_loader);
90 qat_hal_deinit(loader_data->fw_loader); 90 qat_hal_deinit(loader_data->fw_loader);
91
92 if (loader_data->uof_fw)
93 release_firmware(loader_data->uof_fw);
94
95 loader_data->uof_fw = NULL;
91 loader_data->fw_loader = NULL; 96 loader_data->fw_loader = NULL;
92 return 0;
93} 97}
94 98
95int adf_ae_start(struct adf_accel_dev *accel_dev) 99int adf_ae_start(struct adf_accel_dev *accel_dev)
@@ -104,8 +108,9 @@ int adf_ae_start(struct adf_accel_dev *accel_dev)
104 ae_ctr++; 108 ae_ctr++;
105 } 109 }
106 } 110 }
107 pr_info("QAT: qat_dev%d started %d acceleration engines\n", 111 dev_info(&GET_DEV(accel_dev),
108 accel_dev->accel_id, ae_ctr); 112 "qat_dev%d started %d acceleration engines\n",
113 accel_dev->accel_id, ae_ctr);
109 return 0; 114 return 0;
110} 115}
111 116
@@ -121,8 +126,9 @@ int adf_ae_stop(struct adf_accel_dev *accel_dev)
121 ae_ctr++; 126 ae_ctr++;
122 } 127 }
123 } 128 }
124 pr_info("QAT: qat_dev%d stopped %d acceleration engines\n", 129 dev_info(&GET_DEV(accel_dev),
125 accel_dev->accel_id, ae_ctr); 130 "qat_dev%d stopped %d acceleration engines\n",
131 accel_dev->accel_id, ae_ctr);
126 return 0; 132 return 0;
127} 133}
128 134
@@ -147,12 +153,12 @@ int adf_ae_init(struct adf_accel_dev *accel_dev)
147 153
148 accel_dev->fw_loader = loader_data; 154 accel_dev->fw_loader = loader_data;
149 if (qat_hal_init(accel_dev)) { 155 if (qat_hal_init(accel_dev)) {
150 pr_err("QAT: Failed to init the AEs\n"); 156 dev_err(&GET_DEV(accel_dev), "Failed to init the AEs\n");
151 kfree(loader_data); 157 kfree(loader_data);
152 return -EFAULT; 158 return -EFAULT;
153 } 159 }
154 if (adf_ae_reset(accel_dev, 0)) { 160 if (adf_ae_reset(accel_dev, 0)) {
155 pr_err("QAT: Failed to reset the AEs\n"); 161 dev_err(&GET_DEV(accel_dev), "Failed to reset the AEs\n");
156 qat_hal_deinit(loader_data->fw_loader); 162 qat_hal_deinit(loader_data->fw_loader);
157 kfree(loader_data); 163 kfree(loader_data);
158 return -EFAULT; 164 return -EFAULT;
@@ -162,6 +168,9 @@ int adf_ae_init(struct adf_accel_dev *accel_dev)
162 168
163int adf_ae_shutdown(struct adf_accel_dev *accel_dev) 169int adf_ae_shutdown(struct adf_accel_dev *accel_dev)
164{ 170{
171 struct adf_fw_loader_data *loader_data = accel_dev->fw_loader;
172
173 qat_hal_deinit(loader_data->fw_loader);
165 kfree(accel_dev->fw_loader); 174 kfree(accel_dev->fw_loader);
166 accel_dev->fw_loader = NULL; 175 accel_dev->fw_loader = NULL;
167 return 0; 176 return 0;
diff --git a/drivers/crypto/qat/qat_common/adf_aer.c b/drivers/crypto/qat/qat_common/adf_aer.c
index fa1fef824de2..2dbc733b8ab2 100644
--- a/drivers/crypto/qat/qat_common/adf_aer.c
+++ b/drivers/crypto/qat/qat_common/adf_aer.c
@@ -60,14 +60,14 @@ static pci_ers_result_t adf_error_detected(struct pci_dev *pdev,
60{ 60{
61 struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev); 61 struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev);
62 62
63 pr_info("QAT: Acceleration driver hardware error detected.\n"); 63 dev_info(&pdev->dev, "Acceleration driver hardware error detected.\n");
64 if (!accel_dev) { 64 if (!accel_dev) {
65 pr_err("QAT: Can't find acceleration device\n"); 65 dev_err(&pdev->dev, "Can't find acceleration device\n");
66 return PCI_ERS_RESULT_DISCONNECT; 66 return PCI_ERS_RESULT_DISCONNECT;
67 } 67 }
68 68
69 if (state == pci_channel_io_perm_failure) { 69 if (state == pci_channel_io_perm_failure) {
70 pr_err("QAT: Can't recover from device error\n"); 70 dev_err(&pdev->dev, "Can't recover from device error\n");
71 return PCI_ERS_RESULT_DISCONNECT; 71 return PCI_ERS_RESULT_DISCONNECT;
72 } 72 }
73 73
@@ -88,10 +88,12 @@ static void adf_dev_restore(struct adf_accel_dev *accel_dev)
88 struct pci_dev *parent = pdev->bus->self; 88 struct pci_dev *parent = pdev->bus->self;
89 uint16_t bridge_ctl = 0; 89 uint16_t bridge_ctl = 0;
90 90
91 pr_info("QAT: Resetting device qat_dev%d\n", accel_dev->accel_id); 91 dev_info(&GET_DEV(accel_dev), "Resetting device qat_dev%d\n",
92 accel_dev->accel_id);
92 93
93 if (!pci_wait_for_pending_transaction(pdev)) 94 if (!pci_wait_for_pending_transaction(pdev))
94 pr_info("QAT: Transaction still in progress. Proceeding\n"); 95 dev_info(&GET_DEV(accel_dev),
96 "Transaction still in progress. Proceeding\n");
95 97
96 pci_read_config_word(parent, PCI_BRIDGE_CONTROL, &bridge_ctl); 98 pci_read_config_word(parent, PCI_BRIDGE_CONTROL, &bridge_ctl);
97 bridge_ctl |= PCI_BRIDGE_CTL_BUS_RESET; 99 bridge_ctl |= PCI_BRIDGE_CTL_BUS_RESET;
@@ -158,7 +160,8 @@ static int adf_dev_aer_schedule_reset(struct adf_accel_dev *accel_dev,
158 unsigned long timeout = wait_for_completion_timeout( 160 unsigned long timeout = wait_for_completion_timeout(
159 &reset_data->compl, wait_jiffies); 161 &reset_data->compl, wait_jiffies);
160 if (!timeout) { 162 if (!timeout) {
161 pr_err("QAT: Reset device timeout expired\n"); 163 dev_err(&GET_DEV(accel_dev),
164 "Reset device timeout expired\n");
162 ret = -EFAULT; 165 ret = -EFAULT;
163 } 166 }
164 kfree(reset_data); 167 kfree(reset_data);
@@ -184,8 +187,8 @@ static pci_ers_result_t adf_slot_reset(struct pci_dev *pdev)
184 187
185static void adf_resume(struct pci_dev *pdev) 188static void adf_resume(struct pci_dev *pdev)
186{ 189{
187 pr_info("QAT: Acceleration driver reset completed\n"); 190 dev_info(&pdev->dev, "Acceleration driver reset completed\n");
188 pr_info("QAT: Device is up and runnig\n"); 191 dev_info(&pdev->dev, "Device is up and runnig\n");
189} 192}
190 193
191static struct pci_error_handlers adf_err_handler = { 194static struct pci_error_handlers adf_err_handler = {
@@ -236,7 +239,7 @@ EXPORT_SYMBOL_GPL(adf_disable_aer);
236int adf_init_aer(void) 239int adf_init_aer(void)
237{ 240{
238 device_reset_wq = create_workqueue("qat_device_reset_wq"); 241 device_reset_wq = create_workqueue("qat_device_reset_wq");
239 return (device_reset_wq == NULL) ? -EFAULT : 0; 242 return !device_reset_wq ? -EFAULT : 0;
240} 243}
241 244
242void adf_exit_aer(void) 245void adf_exit_aer(void)
diff --git a/drivers/crypto/qat/qat_common/adf_cfg.c b/drivers/crypto/qat/qat_common/adf_cfg.c
index de16da9070a5..ab65bc274561 100644
--- a/drivers/crypto/qat/qat_common/adf_cfg.c
+++ b/drivers/crypto/qat/qat_common/adf_cfg.c
@@ -142,7 +142,8 @@ int adf_cfg_dev_add(struct adf_accel_dev *accel_dev)
142 dev_cfg_data, 142 dev_cfg_data,
143 &qat_dev_cfg_fops); 143 &qat_dev_cfg_fops);
144 if (!dev_cfg_data->debug) { 144 if (!dev_cfg_data->debug) {
145 pr_err("QAT: Failed to create qat cfg debugfs entry.\n"); 145 dev_err(&GET_DEV(accel_dev),
146 "Failed to create qat cfg debugfs entry.\n");
146 kfree(dev_cfg_data); 147 kfree(dev_cfg_data);
147 accel_dev->cfg = NULL; 148 accel_dev->cfg = NULL;
148 return -EFAULT; 149 return -EFAULT;
@@ -305,7 +306,7 @@ int adf_cfg_add_key_value_param(struct adf_accel_dev *accel_dev,
305 snprintf(key_val->val, ADF_CFG_MAX_VAL_LEN_IN_BYTES, 306 snprintf(key_val->val, ADF_CFG_MAX_VAL_LEN_IN_BYTES,
306 "0x%lx", (unsigned long)val); 307 "0x%lx", (unsigned long)val);
307 } else { 308 } else {
308 pr_err("QAT: Unknown type given.\n"); 309 dev_err(&GET_DEV(accel_dev), "Unknown type given.\n");
309 kfree(key_val); 310 kfree(key_val);
310 return -1; 311 return -1;
311 } 312 }
diff --git a/drivers/crypto/qat/qat_common/adf_cfg_strings.h b/drivers/crypto/qat/qat_common/adf_cfg_strings.h
index c7ac758ebc90..13575111382c 100644
--- a/drivers/crypto/qat/qat_common/adf_cfg_strings.h
+++ b/drivers/crypto/qat/qat_common/adf_cfg_strings.h
@@ -59,7 +59,7 @@
59#define ADF_RING_SYM_TX "RingSymTx" 59#define ADF_RING_SYM_TX "RingSymTx"
60#define ADF_RING_RND_TX "RingNrbgTx" 60#define ADF_RING_RND_TX "RingNrbgTx"
61#define ADF_RING_ASYM_RX "RingAsymRx" 61#define ADF_RING_ASYM_RX "RingAsymRx"
62#define ADF_RING_SYM_RX "RinSymRx" 62#define ADF_RING_SYM_RX "RingSymRx"
63#define ADF_RING_RND_RX "RingNrbgRx" 63#define ADF_RING_RND_RX "RingNrbgRx"
64#define ADF_RING_DC_TX "RingTx" 64#define ADF_RING_DC_TX "RingTx"
65#define ADF_RING_DC_RX "RingRx" 65#define ADF_RING_DC_RX "RingRx"
@@ -69,15 +69,15 @@
69#define ADF_DC "Dc" 69#define ADF_DC "Dc"
70#define ADF_ETRMGR_COALESCING_ENABLED "InterruptCoalescingEnabled" 70#define ADF_ETRMGR_COALESCING_ENABLED "InterruptCoalescingEnabled"
71#define ADF_ETRMGR_COALESCING_ENABLED_FORMAT \ 71#define ADF_ETRMGR_COALESCING_ENABLED_FORMAT \
72 ADF_ETRMGR_BANK"%d"ADF_ETRMGR_COALESCING_ENABLED 72 ADF_ETRMGR_BANK "%d" ADF_ETRMGR_COALESCING_ENABLED
73#define ADF_ETRMGR_COALESCE_TIMER "InterruptCoalescingTimerNs" 73#define ADF_ETRMGR_COALESCE_TIMER "InterruptCoalescingTimerNs"
74#define ADF_ETRMGR_COALESCE_TIMER_FORMAT \ 74#define ADF_ETRMGR_COALESCE_TIMER_FORMAT \
75 ADF_ETRMGR_BANK"%d"ADF_ETRMGR_COALESCE_TIMER 75 ADF_ETRMGR_BANK "%d" ADF_ETRMGR_COALESCE_TIMER
76#define ADF_ETRMGR_COALESCING_MSG_ENABLED "InterruptCoalescingNumResponses" 76#define ADF_ETRMGR_COALESCING_MSG_ENABLED "InterruptCoalescingNumResponses"
77#define ADF_ETRMGR_COALESCING_MSG_ENABLED_FORMAT \ 77#define ADF_ETRMGR_COALESCING_MSG_ENABLED_FORMAT \
78 ADF_ETRMGR_BANK"%d"ADF_ETRMGR_COALESCING_MSG_ENABLED 78 ADF_ETRMGR_BANK "%d" ADF_ETRMGR_COALESCING_MSG_ENABLED
79#define ADF_ETRMGR_CORE_AFFINITY "CoreAffinity" 79#define ADF_ETRMGR_CORE_AFFINITY "CoreAffinity"
80#define ADF_ETRMGR_CORE_AFFINITY_FORMAT \ 80#define ADF_ETRMGR_CORE_AFFINITY_FORMAT \
81 ADF_ETRMGR_BANK"%d"ADF_ETRMGR_CORE_AFFINITY 81 ADF_ETRMGR_BANK "%d" ADF_ETRMGR_CORE_AFFINITY
82#define ADF_ACCEL_STR "Accelerator%d" 82#define ADF_ACCEL_STR "Accelerator%d"
83#endif 83#endif
diff --git a/drivers/crypto/qat/qat_common/adf_common_drv.h b/drivers/crypto/qat/qat_common/adf_common_drv.h
index a62e485c8786..0666ee6a3360 100644
--- a/drivers/crypto/qat/qat_common/adf_common_drv.h
+++ b/drivers/crypto/qat/qat_common/adf_common_drv.h
@@ -115,7 +115,7 @@ int adf_dev_restarted_notify(struct adf_accel_dev *accel_dev);
115int adf_ae_init(struct adf_accel_dev *accel_dev); 115int adf_ae_init(struct adf_accel_dev *accel_dev);
116int adf_ae_shutdown(struct adf_accel_dev *accel_dev); 116int adf_ae_shutdown(struct adf_accel_dev *accel_dev);
117int adf_ae_fw_load(struct adf_accel_dev *accel_dev); 117int adf_ae_fw_load(struct adf_accel_dev *accel_dev);
118int adf_ae_fw_release(struct adf_accel_dev *accel_dev); 118void adf_ae_fw_release(struct adf_accel_dev *accel_dev);
119int adf_ae_start(struct adf_accel_dev *accel_dev); 119int adf_ae_start(struct adf_accel_dev *accel_dev);
120int adf_ae_stop(struct adf_accel_dev *accel_dev); 120int adf_ae_stop(struct adf_accel_dev *accel_dev);
121 121
diff --git a/drivers/crypto/qat/qat_common/adf_ctl_drv.c b/drivers/crypto/qat/qat_common/adf_ctl_drv.c
index 74207a6f0516..cb5f066e93a6 100644
--- a/drivers/crypto/qat/qat_common/adf_ctl_drv.c
+++ b/drivers/crypto/qat/qat_common/adf_ctl_drv.c
@@ -77,14 +77,14 @@ struct adf_ctl_drv_info {
77 struct class *drv_class; 77 struct class *drv_class;
78}; 78};
79 79
80static struct adf_ctl_drv_info adt_ctl_drv; 80static struct adf_ctl_drv_info adf_ctl_drv;
81 81
82static void adf_chr_drv_destroy(void) 82static void adf_chr_drv_destroy(void)
83{ 83{
84 device_destroy(adt_ctl_drv.drv_class, MKDEV(adt_ctl_drv.major, 0)); 84 device_destroy(adf_ctl_drv.drv_class, MKDEV(adf_ctl_drv.major, 0));
85 cdev_del(&adt_ctl_drv.drv_cdev); 85 cdev_del(&adf_ctl_drv.drv_cdev);
86 class_destroy(adt_ctl_drv.drv_class); 86 class_destroy(adf_ctl_drv.drv_class);
87 unregister_chrdev_region(MKDEV(adt_ctl_drv.major, 0), 1); 87 unregister_chrdev_region(MKDEV(adf_ctl_drv.major, 0), 1);
88} 88}
89 89
90static int adf_chr_drv_create(void) 90static int adf_chr_drv_create(void)
@@ -97,20 +97,20 @@ static int adf_chr_drv_create(void)
97 return -EFAULT; 97 return -EFAULT;
98 } 98 }
99 99
100 adt_ctl_drv.drv_class = class_create(THIS_MODULE, DEVICE_NAME); 100 adf_ctl_drv.drv_class = class_create(THIS_MODULE, DEVICE_NAME);
101 if (IS_ERR(adt_ctl_drv.drv_class)) { 101 if (IS_ERR(adf_ctl_drv.drv_class)) {
102 pr_err("QAT: class_create failed for adf_ctl\n"); 102 pr_err("QAT: class_create failed for adf_ctl\n");
103 goto err_chrdev_unreg; 103 goto err_chrdev_unreg;
104 } 104 }
105 adt_ctl_drv.major = MAJOR(dev_id); 105 adf_ctl_drv.major = MAJOR(dev_id);
106 cdev_init(&adt_ctl_drv.drv_cdev, &adf_ctl_ops); 106 cdev_init(&adf_ctl_drv.drv_cdev, &adf_ctl_ops);
107 if (cdev_add(&adt_ctl_drv.drv_cdev, dev_id, 1)) { 107 if (cdev_add(&adf_ctl_drv.drv_cdev, dev_id, 1)) {
108 pr_err("QAT: cdev add failed\n"); 108 pr_err("QAT: cdev add failed\n");
109 goto err_class_destr; 109 goto err_class_destr;
110 } 110 }
111 111
112 drv_device = device_create(adt_ctl_drv.drv_class, NULL, 112 drv_device = device_create(adf_ctl_drv.drv_class, NULL,
113 MKDEV(adt_ctl_drv.major, 0), 113 MKDEV(adf_ctl_drv.major, 0),
114 NULL, DEVICE_NAME); 114 NULL, DEVICE_NAME);
115 if (IS_ERR(drv_device)) { 115 if (IS_ERR(drv_device)) {
116 pr_err("QAT: failed to create device\n"); 116 pr_err("QAT: failed to create device\n");
@@ -118,9 +118,9 @@ static int adf_chr_drv_create(void)
118 } 118 }
119 return 0; 119 return 0;
120err_cdev_del: 120err_cdev_del:
121 cdev_del(&adt_ctl_drv.drv_cdev); 121 cdev_del(&adf_ctl_drv.drv_cdev);
122err_class_destr: 122err_class_destr:
123 class_destroy(adt_ctl_drv.drv_class); 123 class_destroy(adf_ctl_drv.drv_class);
124err_chrdev_unreg: 124err_chrdev_unreg:
125 unregister_chrdev_region(dev_id, 1); 125 unregister_chrdev_region(dev_id, 1);
126 return -EFAULT; 126 return -EFAULT;
@@ -159,14 +159,16 @@ static int adf_add_key_value_data(struct adf_accel_dev *accel_dev,
159 if (adf_cfg_add_key_value_param(accel_dev, section, 159 if (adf_cfg_add_key_value_param(accel_dev, section,
160 key_val->key, (void *)val, 160 key_val->key, (void *)val,
161 key_val->type)) { 161 key_val->type)) {
162 pr_err("QAT: failed to add keyvalue.\n"); 162 dev_err(&GET_DEV(accel_dev),
163 "failed to add hex keyvalue.\n");
163 return -EFAULT; 164 return -EFAULT;
164 } 165 }
165 } else { 166 } else {
166 if (adf_cfg_add_key_value_param(accel_dev, section, 167 if (adf_cfg_add_key_value_param(accel_dev, section,
167 key_val->key, key_val->val, 168 key_val->key, key_val->val,
168 key_val->type)) { 169 key_val->type)) {
169 pr_err("QAT: failed to add keyvalue.\n"); 170 dev_err(&GET_DEV(accel_dev),
171 "failed to add keyvalue.\n");
170 return -EFAULT; 172 return -EFAULT;
171 } 173 }
172 } 174 }
@@ -185,12 +187,14 @@ static int adf_copy_key_value_data(struct adf_accel_dev *accel_dev,
185 while (section_head) { 187 while (section_head) {
186 if (copy_from_user(&section, (void __user *)section_head, 188 if (copy_from_user(&section, (void __user *)section_head,
187 sizeof(*section_head))) { 189 sizeof(*section_head))) {
188 pr_err("QAT: failed to copy section info\n"); 190 dev_err(&GET_DEV(accel_dev),
191 "failed to copy section info\n");
189 goto out_err; 192 goto out_err;
190 } 193 }
191 194
192 if (adf_cfg_section_add(accel_dev, section.name)) { 195 if (adf_cfg_section_add(accel_dev, section.name)) {
193 pr_err("QAT: failed to add section.\n"); 196 dev_err(&GET_DEV(accel_dev),
197 "failed to add section.\n");
194 goto out_err; 198 goto out_err;
195 } 199 }
196 200
@@ -199,7 +203,8 @@ static int adf_copy_key_value_data(struct adf_accel_dev *accel_dev,
199 while (params_head) { 203 while (params_head) {
200 if (copy_from_user(&key_val, (void __user *)params_head, 204 if (copy_from_user(&key_val, (void __user *)params_head,
201 sizeof(key_val))) { 205 sizeof(key_val))) {
202 pr_err("QAT: Failed to copy keyvalue.\n"); 206 dev_err(&GET_DEV(accel_dev),
207 "Failed to copy keyvalue.\n");
203 goto out_err; 208 goto out_err;
204 } 209 }
205 if (adf_add_key_value_data(accel_dev, section.name, 210 if (adf_add_key_value_data(accel_dev, section.name,
@@ -258,8 +263,9 @@ static int adf_ctl_is_device_in_use(int id)
258 263
259 if (id == dev->accel_id || id == ADF_CFG_ALL_DEVICES) { 264 if (id == dev->accel_id || id == ADF_CFG_ALL_DEVICES) {
260 if (adf_devmgr_in_reset(dev) || adf_dev_in_use(dev)) { 265 if (adf_devmgr_in_reset(dev) || adf_dev_in_use(dev)) {
261 pr_info("QAT: device qat_dev%d is busy\n", 266 dev_info(&GET_DEV(dev),
262 dev->accel_id); 267 "device qat_dev%d is busy\n",
268 dev->accel_id);
263 return -EBUSY; 269 return -EBUSY;
264 } 270 }
265 } 271 }
@@ -280,7 +286,8 @@ static int adf_ctl_stop_devices(uint32_t id)
280 continue; 286 continue;
281 287
282 if (adf_dev_stop(accel_dev)) { 288 if (adf_dev_stop(accel_dev)) {
283 pr_err("QAT: Failed to stop qat_dev%d\n", id); 289 dev_err(&GET_DEV(accel_dev),
290 "Failed to stop qat_dev%d\n", id);
284 ret = -EFAULT; 291 ret = -EFAULT;
285 } else { 292 } else {
286 adf_dev_shutdown(accel_dev); 293 adf_dev_shutdown(accel_dev);
@@ -343,17 +350,20 @@ static int adf_ctl_ioctl_dev_start(struct file *fp, unsigned int cmd,
343 } 350 }
344 351
345 if (!adf_dev_started(accel_dev)) { 352 if (!adf_dev_started(accel_dev)) {
346 pr_info("QAT: Starting acceleration device qat_dev%d.\n", 353 dev_info(&GET_DEV(accel_dev),
347 ctl_data->device_id); 354 "Starting acceleration device qat_dev%d.\n",
355 ctl_data->device_id);
348 ret = adf_dev_init(accel_dev); 356 ret = adf_dev_init(accel_dev);
349 if (!ret) 357 if (!ret)
350 ret = adf_dev_start(accel_dev); 358 ret = adf_dev_start(accel_dev);
351 } else { 359 } else {
352 pr_info("QAT: Acceleration device qat_dev%d already started.\n", 360 dev_info(&GET_DEV(accel_dev),
353 ctl_data->device_id); 361 "Acceleration device qat_dev%d already started.\n",
362 ctl_data->device_id);
354 } 363 }
355 if (ret) { 364 if (ret) {
356 pr_err("QAT: Failed to start qat_dev%d\n", ctl_data->device_id); 365 dev_err(&GET_DEV(accel_dev), "Failed to start qat_dev%d\n",
366 ctl_data->device_id);
357 adf_dev_stop(accel_dev); 367 adf_dev_stop(accel_dev);
358 adf_dev_shutdown(accel_dev); 368 adf_dev_shutdown(accel_dev);
359 } 369 }
@@ -408,7 +418,7 @@ static int adf_ctl_ioctl_get_status(struct file *fp, unsigned int cmd,
408 418
409 if (copy_to_user((void __user *)arg, &dev_info, 419 if (copy_to_user((void __user *)arg, &dev_info,
410 sizeof(struct adf_dev_status_info))) { 420 sizeof(struct adf_dev_status_info))) {
411 pr_err("QAT: failed to copy status.\n"); 421 dev_err(&GET_DEV(accel_dev), "failed to copy status.\n");
412 return -EFAULT; 422 return -EFAULT;
413 } 423 }
414 return 0; 424 return 0;
diff --git a/drivers/crypto/qat/qat_common/adf_dev_mgr.c b/drivers/crypto/qat/qat_common/adf_dev_mgr.c
index 4a0a829d4500..3f0ff9e7d840 100644
--- a/drivers/crypto/qat/qat_common/adf_dev_mgr.c
+++ b/drivers/crypto/qat/qat_common/adf_dev_mgr.c
@@ -67,7 +67,8 @@ int adf_devmgr_add_dev(struct adf_accel_dev *accel_dev)
67 struct list_head *itr; 67 struct list_head *itr;
68 68
69 if (num_devices == ADF_MAX_DEVICES) { 69 if (num_devices == ADF_MAX_DEVICES) {
70 pr_err("QAT: Only support up to %d devices\n", ADF_MAX_DEVICES); 70 dev_err(&GET_DEV(accel_dev), "Only support up to %d devices\n",
71 ADF_MAX_DEVICES);
71 return -EFAULT; 72 return -EFAULT;
72 } 73 }
73 74
diff --git a/drivers/crypto/qat/qat_common/adf_init.c b/drivers/crypto/qat/qat_common/adf_init.c
index 8f0ca498ab87..245f43237a2d 100644
--- a/drivers/crypto/qat/qat_common/adf_init.c
+++ b/drivers/crypto/qat/qat_common/adf_init.c
@@ -124,12 +124,12 @@ int adf_dev_init(struct adf_accel_dev *accel_dev)
124 124
125 if (!hw_data) { 125 if (!hw_data) {
126 dev_err(&GET_DEV(accel_dev), 126 dev_err(&GET_DEV(accel_dev),
127 "QAT: Failed to init device - hw_data not set\n"); 127 "Failed to init device - hw_data not set\n");
128 return -EFAULT; 128 return -EFAULT;
129 } 129 }
130 130
131 if (!test_bit(ADF_STATUS_CONFIGURED, &accel_dev->status)) { 131 if (!test_bit(ADF_STATUS_CONFIGURED, &accel_dev->status)) {
132 pr_info("QAT: Device not configured\n"); 132 dev_err(&GET_DEV(accel_dev), "Device not configured\n");
133 return -EFAULT; 133 return -EFAULT;
134 } 134 }
135 135
@@ -151,20 +151,21 @@ int adf_dev_init(struct adf_accel_dev *accel_dev)
151 hw_data->enable_ints(accel_dev); 151 hw_data->enable_ints(accel_dev);
152 152
153 if (adf_ae_init(accel_dev)) { 153 if (adf_ae_init(accel_dev)) {
154 pr_err("QAT: Failed to initialise Acceleration Engine\n"); 154 dev_err(&GET_DEV(accel_dev),
155 "Failed to initialise Acceleration Engine\n");
155 return -EFAULT; 156 return -EFAULT;
156 } 157 }
157 set_bit(ADF_STATUS_AE_INITIALISED, &accel_dev->status); 158 set_bit(ADF_STATUS_AE_INITIALISED, &accel_dev->status);
158 159
159 if (adf_ae_fw_load(accel_dev)) { 160 if (adf_ae_fw_load(accel_dev)) {
160 pr_err("QAT: Failed to load acceleration FW\n"); 161 dev_err(&GET_DEV(accel_dev),
161 adf_ae_fw_release(accel_dev); 162 "Failed to load acceleration FW\n");
162 return -EFAULT; 163 return -EFAULT;
163 } 164 }
164 set_bit(ADF_STATUS_AE_UCODE_LOADED, &accel_dev->status); 165 set_bit(ADF_STATUS_AE_UCODE_LOADED, &accel_dev->status);
165 166
166 if (hw_data->alloc_irq(accel_dev)) { 167 if (hw_data->alloc_irq(accel_dev)) {
167 pr_err("QAT: Failed to allocate interrupts\n"); 168 dev_err(&GET_DEV(accel_dev), "Failed to allocate interrupts\n");
168 return -EFAULT; 169 return -EFAULT;
169 } 170 }
170 set_bit(ADF_STATUS_IRQ_ALLOCATED, &accel_dev->status); 171 set_bit(ADF_STATUS_IRQ_ALLOCATED, &accel_dev->status);
@@ -179,8 +180,9 @@ int adf_dev_init(struct adf_accel_dev *accel_dev)
179 if (!service->admin) 180 if (!service->admin)
180 continue; 181 continue;
181 if (service->event_hld(accel_dev, ADF_EVENT_INIT)) { 182 if (service->event_hld(accel_dev, ADF_EVENT_INIT)) {
182 pr_err("QAT: Failed to initialise service %s\n", 183 dev_err(&GET_DEV(accel_dev),
183 service->name); 184 "Failed to initialise service %s\n",
185 service->name);
184 return -EFAULT; 186 return -EFAULT;
185 } 187 }
186 set_bit(accel_dev->accel_id, &service->init_status); 188 set_bit(accel_dev->accel_id, &service->init_status);
@@ -190,8 +192,9 @@ int adf_dev_init(struct adf_accel_dev *accel_dev)
190 if (service->admin) 192 if (service->admin)
191 continue; 193 continue;
192 if (service->event_hld(accel_dev, ADF_EVENT_INIT)) { 194 if (service->event_hld(accel_dev, ADF_EVENT_INIT)) {
193 pr_err("QAT: Failed to initialise service %s\n", 195 dev_err(&GET_DEV(accel_dev),
194 service->name); 196 "Failed to initialise service %s\n",
197 service->name);
195 return -EFAULT; 198 return -EFAULT;
196 } 199 }
197 set_bit(accel_dev->accel_id, &service->init_status); 200 set_bit(accel_dev->accel_id, &service->init_status);
@@ -221,7 +224,7 @@ int adf_dev_start(struct adf_accel_dev *accel_dev)
221 set_bit(ADF_STATUS_STARTING, &accel_dev->status); 224 set_bit(ADF_STATUS_STARTING, &accel_dev->status);
222 225
223 if (adf_ae_start(accel_dev)) { 226 if (adf_ae_start(accel_dev)) {
224 pr_err("QAT: AE Start Failed\n"); 227 dev_err(&GET_DEV(accel_dev), "AE Start Failed\n");
225 return -EFAULT; 228 return -EFAULT;
226 } 229 }
227 set_bit(ADF_STATUS_AE_STARTED, &accel_dev->status); 230 set_bit(ADF_STATUS_AE_STARTED, &accel_dev->status);
@@ -231,8 +234,9 @@ int adf_dev_start(struct adf_accel_dev *accel_dev)
231 if (!service->admin) 234 if (!service->admin)
232 continue; 235 continue;
233 if (service->event_hld(accel_dev, ADF_EVENT_START)) { 236 if (service->event_hld(accel_dev, ADF_EVENT_START)) {
234 pr_err("QAT: Failed to start service %s\n", 237 dev_err(&GET_DEV(accel_dev),
235 service->name); 238 "Failed to start service %s\n",
239 service->name);
236 return -EFAULT; 240 return -EFAULT;
237 } 241 }
238 set_bit(accel_dev->accel_id, &service->start_status); 242 set_bit(accel_dev->accel_id, &service->start_status);
@@ -242,8 +246,9 @@ int adf_dev_start(struct adf_accel_dev *accel_dev)
242 if (service->admin) 246 if (service->admin)
243 continue; 247 continue;
244 if (service->event_hld(accel_dev, ADF_EVENT_START)) { 248 if (service->event_hld(accel_dev, ADF_EVENT_START)) {
245 pr_err("QAT: Failed to start service %s\n", 249 dev_err(&GET_DEV(accel_dev),
246 service->name); 250 "Failed to start service %s\n",
251 service->name);
247 return -EFAULT; 252 return -EFAULT;
248 } 253 }
249 set_bit(accel_dev->accel_id, &service->start_status); 254 set_bit(accel_dev->accel_id, &service->start_status);
@@ -253,7 +258,8 @@ int adf_dev_start(struct adf_accel_dev *accel_dev)
253 set_bit(ADF_STATUS_STARTED, &accel_dev->status); 258 set_bit(ADF_STATUS_STARTED, &accel_dev->status);
254 259
255 if (qat_algs_register()) { 260 if (qat_algs_register()) {
256 pr_err("QAT: Failed to register crypto algs\n"); 261 dev_err(&GET_DEV(accel_dev),
262 "Failed to register crypto algs\n");
257 set_bit(ADF_STATUS_STARTING, &accel_dev->status); 263 set_bit(ADF_STATUS_STARTING, &accel_dev->status);
258 clear_bit(ADF_STATUS_STARTED, &accel_dev->status); 264 clear_bit(ADF_STATUS_STARTED, &accel_dev->status);
259 return -EFAULT; 265 return -EFAULT;
@@ -287,7 +293,8 @@ int adf_dev_stop(struct adf_accel_dev *accel_dev)
287 clear_bit(ADF_STATUS_STARTED, &accel_dev->status); 293 clear_bit(ADF_STATUS_STARTED, &accel_dev->status);
288 294
289 if (qat_algs_unregister()) 295 if (qat_algs_unregister())
290 pr_err("QAT: Failed to unregister crypto algs\n"); 296 dev_err(&GET_DEV(accel_dev),
297 "Failed to unregister crypto algs\n");
291 298
292 list_for_each(list_itr, &service_table) { 299 list_for_each(list_itr, &service_table) {
293 service = list_entry(list_itr, struct service_hndl, list); 300 service = list_entry(list_itr, struct service_hndl, list);
@@ -310,8 +317,9 @@ int adf_dev_stop(struct adf_accel_dev *accel_dev)
310 if (!test_bit(accel_dev->accel_id, &service->start_status)) 317 if (!test_bit(accel_dev->accel_id, &service->start_status))
311 continue; 318 continue;
312 if (service->event_hld(accel_dev, ADF_EVENT_STOP)) 319 if (service->event_hld(accel_dev, ADF_EVENT_STOP))
313 pr_err("QAT: Failed to shutdown service %s\n", 320 dev_err(&GET_DEV(accel_dev),
314 service->name); 321 "Failed to shutdown service %s\n",
322 service->name);
315 else 323 else
316 clear_bit(accel_dev->accel_id, &service->start_status); 324 clear_bit(accel_dev->accel_id, &service->start_status);
317 } 325 }
@@ -321,7 +329,7 @@ int adf_dev_stop(struct adf_accel_dev *accel_dev)
321 329
322 if (test_bit(ADF_STATUS_AE_STARTED, &accel_dev->status)) { 330 if (test_bit(ADF_STATUS_AE_STARTED, &accel_dev->status)) {
323 if (adf_ae_stop(accel_dev)) 331 if (adf_ae_stop(accel_dev))
324 pr_err("QAT: failed to stop AE\n"); 332 dev_err(&GET_DEV(accel_dev), "failed to stop AE\n");
325 else 333 else
326 clear_bit(ADF_STATUS_AE_STARTED, &accel_dev->status); 334 clear_bit(ADF_STATUS_AE_STARTED, &accel_dev->status);
327 } 335 }
@@ -350,16 +358,14 @@ void adf_dev_shutdown(struct adf_accel_dev *accel_dev)
350 } 358 }
351 359
352 if (test_bit(ADF_STATUS_AE_UCODE_LOADED, &accel_dev->status)) { 360 if (test_bit(ADF_STATUS_AE_UCODE_LOADED, &accel_dev->status)) {
353 if (adf_ae_fw_release(accel_dev)) 361 adf_ae_fw_release(accel_dev);
354 pr_err("QAT: Failed to release the ucode\n"); 362 clear_bit(ADF_STATUS_AE_UCODE_LOADED, &accel_dev->status);
355 else
356 clear_bit(ADF_STATUS_AE_UCODE_LOADED,
357 &accel_dev->status);
358 } 363 }
359 364
360 if (test_bit(ADF_STATUS_AE_INITIALISED, &accel_dev->status)) { 365 if (test_bit(ADF_STATUS_AE_INITIALISED, &accel_dev->status)) {
361 if (adf_ae_shutdown(accel_dev)) 366 if (adf_ae_shutdown(accel_dev))
362 pr_err("QAT: Failed to shutdown Accel Engine\n"); 367 dev_err(&GET_DEV(accel_dev),
368 "Failed to shutdown Accel Engine\n");
363 else 369 else
364 clear_bit(ADF_STATUS_AE_INITIALISED, 370 clear_bit(ADF_STATUS_AE_INITIALISED,
365 &accel_dev->status); 371 &accel_dev->status);
@@ -372,8 +378,9 @@ void adf_dev_shutdown(struct adf_accel_dev *accel_dev)
372 if (!test_bit(accel_dev->accel_id, &service->init_status)) 378 if (!test_bit(accel_dev->accel_id, &service->init_status))
373 continue; 379 continue;
374 if (service->event_hld(accel_dev, ADF_EVENT_SHUTDOWN)) 380 if (service->event_hld(accel_dev, ADF_EVENT_SHUTDOWN))
375 pr_err("QAT: Failed to shutdown service %s\n", 381 dev_err(&GET_DEV(accel_dev),
376 service->name); 382 "Failed to shutdown service %s\n",
383 service->name);
377 else 384 else
378 clear_bit(accel_dev->accel_id, &service->init_status); 385 clear_bit(accel_dev->accel_id, &service->init_status);
379 } 386 }
@@ -384,8 +391,9 @@ void adf_dev_shutdown(struct adf_accel_dev *accel_dev)
384 if (!test_bit(accel_dev->accel_id, &service->init_status)) 391 if (!test_bit(accel_dev->accel_id, &service->init_status))
385 continue; 392 continue;
386 if (service->event_hld(accel_dev, ADF_EVENT_SHUTDOWN)) 393 if (service->event_hld(accel_dev, ADF_EVENT_SHUTDOWN))
387 pr_err("QAT: Failed to shutdown service %s\n", 394 dev_err(&GET_DEV(accel_dev),
388 service->name); 395 "Failed to shutdown service %s\n",
396 service->name);
389 else 397 else
390 clear_bit(accel_dev->accel_id, &service->init_status); 398 clear_bit(accel_dev->accel_id, &service->init_status);
391 } 399 }
@@ -419,16 +427,18 @@ int adf_dev_restarting_notify(struct adf_accel_dev *accel_dev)
419 if (service->admin) 427 if (service->admin)
420 continue; 428 continue;
421 if (service->event_hld(accel_dev, ADF_EVENT_RESTARTING)) 429 if (service->event_hld(accel_dev, ADF_EVENT_RESTARTING))
422 pr_err("QAT: Failed to restart service %s.\n", 430 dev_err(&GET_DEV(accel_dev),
423 service->name); 431 "Failed to restart service %s.\n",
432 service->name);
424 } 433 }
425 list_for_each(list_itr, &service_table) { 434 list_for_each(list_itr, &service_table) {
426 service = list_entry(list_itr, struct service_hndl, list); 435 service = list_entry(list_itr, struct service_hndl, list);
427 if (!service->admin) 436 if (!service->admin)
428 continue; 437 continue;
429 if (service->event_hld(accel_dev, ADF_EVENT_RESTARTING)) 438 if (service->event_hld(accel_dev, ADF_EVENT_RESTARTING))
430 pr_err("QAT: Failed to restart service %s.\n", 439 dev_err(&GET_DEV(accel_dev),
431 service->name); 440 "Failed to restart service %s.\n",
441 service->name);
432 } 442 }
433 return 0; 443 return 0;
434} 444}
@@ -443,16 +453,18 @@ int adf_dev_restarted_notify(struct adf_accel_dev *accel_dev)
443 if (service->admin) 453 if (service->admin)
444 continue; 454 continue;
445 if (service->event_hld(accel_dev, ADF_EVENT_RESTARTED)) 455 if (service->event_hld(accel_dev, ADF_EVENT_RESTARTED))
446 pr_err("QAT: Failed to restart service %s.\n", 456 dev_err(&GET_DEV(accel_dev),
447 service->name); 457 "Failed to restart service %s.\n",
458 service->name);
448 } 459 }
449 list_for_each(list_itr, &service_table) { 460 list_for_each(list_itr, &service_table) {
450 service = list_entry(list_itr, struct service_hndl, list); 461 service = list_entry(list_itr, struct service_hndl, list);
451 if (!service->admin) 462 if (!service->admin)
452 continue; 463 continue;
453 if (service->event_hld(accel_dev, ADF_EVENT_RESTARTED)) 464 if (service->event_hld(accel_dev, ADF_EVENT_RESTARTED))
454 pr_err("QAT: Failed to restart service %s.\n", 465 dev_err(&GET_DEV(accel_dev),
455 service->name); 466 "Failed to restart service %s.\n",
467 service->name);
456 } 468 }
457 return 0; 469 return 0;
458} 470}
diff --git a/drivers/crypto/qat/qat_common/adf_transport.c b/drivers/crypto/qat/qat_common/adf_transport.c
index 7dd54aaee9fa..ccec327489da 100644
--- a/drivers/crypto/qat/qat_common/adf_transport.c
+++ b/drivers/crypto/qat/qat_common/adf_transport.c
@@ -195,7 +195,7 @@ static int adf_init_ring(struct adf_etr_ring_data *ring)
195 memset(ring->base_addr, 0x7F, ring_size_bytes); 195 memset(ring->base_addr, 0x7F, ring_size_bytes);
196 /* The base_addr has to be aligned to the size of the buffer */ 196 /* The base_addr has to be aligned to the size of the buffer */
197 if (adf_check_ring_alignment(ring->dma_addr, ring_size_bytes)) { 197 if (adf_check_ring_alignment(ring->dma_addr, ring_size_bytes)) {
198 pr_err("QAT: Ring address not aligned\n"); 198 dev_err(&GET_DEV(accel_dev), "Ring address not aligned\n");
199 dma_free_coherent(&GET_DEV(accel_dev), ring_size_bytes, 199 dma_free_coherent(&GET_DEV(accel_dev), ring_size_bytes,
200 ring->base_addr, ring->dma_addr); 200 ring->base_addr, ring->dma_addr);
201 return -EFAULT; 201 return -EFAULT;
@@ -242,32 +242,33 @@ int adf_create_ring(struct adf_accel_dev *accel_dev, const char *section,
242 int ret; 242 int ret;
243 243
244 if (bank_num >= GET_MAX_BANKS(accel_dev)) { 244 if (bank_num >= GET_MAX_BANKS(accel_dev)) {
245 pr_err("QAT: Invalid bank number\n"); 245 dev_err(&GET_DEV(accel_dev), "Invalid bank number\n");
246 return -EFAULT; 246 return -EFAULT;
247 } 247 }
248 if (msg_size > ADF_MSG_SIZE_TO_BYTES(ADF_MAX_MSG_SIZE)) { 248 if (msg_size > ADF_MSG_SIZE_TO_BYTES(ADF_MAX_MSG_SIZE)) {
249 pr_err("QAT: Invalid msg size\n"); 249 dev_err(&GET_DEV(accel_dev), "Invalid msg size\n");
250 return -EFAULT; 250 return -EFAULT;
251 } 251 }
252 if (ADF_MAX_INFLIGHTS(adf_verify_ring_size(msg_size, num_msgs), 252 if (ADF_MAX_INFLIGHTS(adf_verify_ring_size(msg_size, num_msgs),
253 ADF_BYTES_TO_MSG_SIZE(msg_size)) < 2) { 253 ADF_BYTES_TO_MSG_SIZE(msg_size)) < 2) {
254 pr_err("QAT: Invalid ring size for given msg size\n"); 254 dev_err(&GET_DEV(accel_dev),
255 "Invalid ring size for given msg size\n");
255 return -EFAULT; 256 return -EFAULT;
256 } 257 }
257 if (adf_cfg_get_param_value(accel_dev, section, ring_name, val)) { 258 if (adf_cfg_get_param_value(accel_dev, section, ring_name, val)) {
258 pr_err("QAT: Section %s, no such entry : %s\n", 259 dev_err(&GET_DEV(accel_dev), "Section %s, no such entry : %s\n",
259 section, ring_name); 260 section, ring_name);
260 return -EFAULT; 261 return -EFAULT;
261 } 262 }
262 if (kstrtouint(val, 10, &ring_num)) { 263 if (kstrtouint(val, 10, &ring_num)) {
263 pr_err("QAT: Can't get ring number\n"); 264 dev_err(&GET_DEV(accel_dev), "Can't get ring number\n");
264 return -EFAULT; 265 return -EFAULT;
265 } 266 }
266 267
267 bank = &transport_data->banks[bank_num]; 268 bank = &transport_data->banks[bank_num];
268 if (adf_reserve_ring(bank, ring_num)) { 269 if (adf_reserve_ring(bank, ring_num)) {
269 pr_err("QAT: Ring %d, %s already exists.\n", 270 dev_err(&GET_DEV(accel_dev), "Ring %d, %s already exists.\n",
270 ring_num, ring_name); 271 ring_num, ring_name);
271 return -EFAULT; 272 return -EFAULT;
272 } 273 }
273 ring = &bank->rings[ring_num]; 274 ring = &bank->rings[ring_num];
@@ -287,7 +288,8 @@ int adf_create_ring(struct adf_accel_dev *accel_dev, const char *section,
287 accel_dev->hw_device->hw_arb_ring_enable(ring); 288 accel_dev->hw_device->hw_arb_ring_enable(ring);
288 289
289 if (adf_ring_debugfs_add(ring, ring_name)) { 290 if (adf_ring_debugfs_add(ring, ring_name)) {
290 pr_err("QAT: Couldn't add ring debugfs entry\n"); 291 dev_err(&GET_DEV(accel_dev),
292 "Couldn't add ring debugfs entry\n");
291 ret = -EFAULT; 293 ret = -EFAULT;
292 goto err; 294 goto err;
293 } 295 }
@@ -428,7 +430,8 @@ static int adf_init_bank(struct adf_accel_dev *accel_dev,
428 goto err; 430 goto err;
429 } else { 431 } else {
430 if (i < hw_data->tx_rx_gap) { 432 if (i < hw_data->tx_rx_gap) {
431 pr_err("QAT: Invalid tx rings mask config\n"); 433 dev_err(&GET_DEV(accel_dev),
434 "Invalid tx rings mask config\n");
432 goto err; 435 goto err;
433 } 436 }
434 tx_ring = &bank->rings[i - hw_data->tx_rx_gap]; 437 tx_ring = &bank->rings[i - hw_data->tx_rx_gap];
@@ -436,7 +439,8 @@ static int adf_init_bank(struct adf_accel_dev *accel_dev,
436 } 439 }
437 } 440 }
438 if (adf_bank_debugfs_add(bank)) { 441 if (adf_bank_debugfs_add(bank)) {
439 pr_err("QAT: Failed to add bank debugfs entry\n"); 442 dev_err(&GET_DEV(accel_dev),
443 "Failed to add bank debugfs entry\n");
440 goto err; 444 goto err;
441 } 445 }
442 446
@@ -492,7 +496,8 @@ int adf_init_etr_data(struct adf_accel_dev *accel_dev)
492 etr_data->debug = debugfs_create_dir("transport", 496 etr_data->debug = debugfs_create_dir("transport",
493 accel_dev->debugfs_dir); 497 accel_dev->debugfs_dir);
494 if (!etr_data->debug) { 498 if (!etr_data->debug) {
495 pr_err("QAT: Unable to create transport debugfs entry\n"); 499 dev_err(&GET_DEV(accel_dev),
500 "Unable to create transport debugfs entry\n");
496 ret = -ENOENT; 501 ret = -ENOENT;
497 goto err_bank_debug; 502 goto err_bank_debug;
498 } 503 }
diff --git a/drivers/crypto/qat/qat_common/adf_transport_debug.c b/drivers/crypto/qat/qat_common/adf_transport_debug.c
index 6b6974553514..e41986967294 100644
--- a/drivers/crypto/qat/qat_common/adf_transport_debug.c
+++ b/drivers/crypto/qat/qat_common/adf_transport_debug.c
@@ -100,6 +100,8 @@ static int adf_ring_show(struct seq_file *sfile, void *v)
100 empty = READ_CSR_E_STAT(csr, bank->bank_number); 100 empty = READ_CSR_E_STAT(csr, bank->bank_number);
101 101
102 seq_puts(sfile, "------- Ring configuration -------\n"); 102 seq_puts(sfile, "------- Ring configuration -------\n");
103 seq_printf(sfile, "ring name: %s\n",
104 ring->ring_debug->ring_name);
103 seq_printf(sfile, "ring num %d, bank num %d\n", 105 seq_printf(sfile, "ring num %d, bank num %d\n",
104 ring->ring_number, ring->bank->bank_number); 106 ring->ring_number, ring->bank->bank_number);
105 seq_printf(sfile, "head %x, tail %x, empty: %d\n", 107 seq_printf(sfile, "head %x, tail %x, empty: %d\n",
diff --git a/drivers/crypto/qat/qat_common/icp_qat_hw.h b/drivers/crypto/qat/qat_common/icp_qat_hw.h
index 68f191b653b0..121d5e6e46ca 100644
--- a/drivers/crypto/qat/qat_common/icp_qat_hw.h
+++ b/drivers/crypto/qat/qat_common/icp_qat_hw.h
@@ -145,7 +145,7 @@ struct icp_qat_hw_auth_setup {
145}; 145};
146 146
147#define QAT_HW_DEFAULT_ALIGNMENT 8 147#define QAT_HW_DEFAULT_ALIGNMENT 8
148#define QAT_HW_ROUND_UP(val, n) (((val) + ((n)-1)) & (~(n-1))) 148#define QAT_HW_ROUND_UP(val, n) (((val) + ((n) - 1)) & (~(n - 1)))
149#define ICP_QAT_HW_NULL_STATE1_SZ 32 149#define ICP_QAT_HW_NULL_STATE1_SZ 32
150#define ICP_QAT_HW_MD5_STATE1_SZ 16 150#define ICP_QAT_HW_MD5_STATE1_SZ 16
151#define ICP_QAT_HW_SHA1_STATE1_SZ 20 151#define ICP_QAT_HW_SHA1_STATE1_SZ 20
diff --git a/drivers/crypto/qat/qat_common/qat_crypto.c b/drivers/crypto/qat/qat_common/qat_crypto.c
index 828f2a686aab..3bd705ca5973 100644
--- a/drivers/crypto/qat/qat_common/qat_crypto.c
+++ b/drivers/crypto/qat/qat_common/qat_crypto.c
@@ -110,13 +110,13 @@ struct qat_crypto_instance *qat_crypto_get_instance_node(int node)
110 list_for_each(itr, adf_devmgr_get_head()) { 110 list_for_each(itr, adf_devmgr_get_head()) {
111 accel_dev = list_entry(itr, struct adf_accel_dev, list); 111 accel_dev = list_entry(itr, struct adf_accel_dev, list);
112 if ((node == dev_to_node(&GET_DEV(accel_dev)) || 112 if ((node == dev_to_node(&GET_DEV(accel_dev)) ||
113 dev_to_node(&GET_DEV(accel_dev)) < 0) 113 dev_to_node(&GET_DEV(accel_dev)) < 0) &&
114 && adf_dev_started(accel_dev)) 114 adf_dev_started(accel_dev))
115 break; 115 break;
116 accel_dev = NULL; 116 accel_dev = NULL;
117 } 117 }
118 if (!accel_dev) { 118 if (!accel_dev) {
119 pr_err("QAT: Could not find device on node %d\n", node); 119 pr_err("QAT: Could not find a device on node %d\n", node);
120 accel_dev = adf_devmgr_get_first(); 120 accel_dev = adf_devmgr_get_first();
121 } 121 }
122 if (!accel_dev || !adf_dev_started(accel_dev)) 122 if (!accel_dev || !adf_dev_started(accel_dev))
@@ -137,7 +137,8 @@ struct qat_crypto_instance *qat_crypto_get_instance_node(int node)
137 if (atomic_add_return(1, &inst_best->refctr) == 1) { 137 if (atomic_add_return(1, &inst_best->refctr) == 1) {
138 if (adf_dev_get(accel_dev)) { 138 if (adf_dev_get(accel_dev)) {
139 atomic_dec(&inst_best->refctr); 139 atomic_dec(&inst_best->refctr);
140 pr_err("QAT: Could increment dev refctr\n"); 140 dev_err(&GET_DEV(accel_dev),
141 "Could not increment dev refctr\n");
141 return NULL; 142 return NULL;
142 } 143 }
143 } 144 }
diff --git a/drivers/crypto/qat/qat_common/qat_hal.c b/drivers/crypto/qat/qat_common/qat_hal.c
index b818c19713bf..274ff7e9de6e 100644
--- a/drivers/crypto/qat/qat_common/qat_hal.c
+++ b/drivers/crypto/qat/qat_common/qat_hal.c
@@ -434,8 +434,8 @@ static void qat_hal_reset_timestamp(struct icp_qat_fw_loader_handle *handle)
434 SET_GLB_CSR(handle, MISC_CONTROL, misc_ctl | MC_TIMESTAMP_ENABLE); 434 SET_GLB_CSR(handle, MISC_CONTROL, misc_ctl | MC_TIMESTAMP_ENABLE);
435} 435}
436 436
437#define ESRAM_AUTO_TINIT (1<<2) 437#define ESRAM_AUTO_TINIT BIT(2)
438#define ESRAM_AUTO_TINIT_DONE (1<<3) 438#define ESRAM_AUTO_TINIT_DONE BIT(3)
439#define ESRAM_AUTO_INIT_USED_CYCLES (1640) 439#define ESRAM_AUTO_INIT_USED_CYCLES (1640)
440#define ESRAM_AUTO_INIT_CSR_OFFSET 0xC1C 440#define ESRAM_AUTO_INIT_CSR_OFFSET 0xC1C
441static int qat_hal_init_esram(struct icp_qat_fw_loader_handle *handle) 441static int qat_hal_init_esram(struct icp_qat_fw_loader_handle *handle)
@@ -718,7 +718,7 @@ int qat_hal_init(struct adf_accel_dev *accel_dev)
718 handle->hal_handle->ae_max_num = max_en_ae_id + 1; 718 handle->hal_handle->ae_max_num = max_en_ae_id + 1;
719 /* take all AEs out of reset */ 719 /* take all AEs out of reset */
720 if (qat_hal_clr_reset(handle)) { 720 if (qat_hal_clr_reset(handle)) {
721 pr_err("QAT: qat_hal_clr_reset error\n"); 721 dev_err(&GET_DEV(accel_dev), "qat_hal_clr_reset error\n");
722 goto out_err; 722 goto out_err;
723 } 723 }
724 if (qat_hal_clear_gpr(handle)) 724 if (qat_hal_clear_gpr(handle))
diff --git a/drivers/crypto/qat/qat_dh895xcc/adf_admin.c b/drivers/crypto/qat/qat_dh895xcc/adf_admin.c
index 53c491b59f07..e4666065c399 100644
--- a/drivers/crypto/qat/qat_dh895xcc/adf_admin.c
+++ b/drivers/crypto/qat/qat_dh895xcc/adf_admin.c
@@ -93,7 +93,8 @@ int adf_put_admin_msg_sync(struct adf_accel_dev *accel_dev,
93 memcpy(out, admin->virt_addr + offset + 93 memcpy(out, admin->virt_addr + offset +
94 ADF_ADMINMSG_LEN, ADF_ADMINMSG_LEN); 94 ADF_ADMINMSG_LEN, ADF_ADMINMSG_LEN);
95 else 95 else
96 pr_err("QAT: Failed to send admin msg to accelerator\n"); 96 dev_err(&GET_DEV(accel_dev),
97 "Failed to send admin msg to accelerator\n");
97 98
98 mutex_unlock(&admin->lock); 99 mutex_unlock(&admin->lock);
99 return received ? 0 : -EFAULT; 100 return received ? 0 : -EFAULT;
diff --git a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c
index 6a735d5c0e37..b1386922d7a2 100644
--- a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c
+++ b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c
@@ -150,7 +150,8 @@ void adf_get_arbiter_mapping(struct adf_accel_dev *accel_dev,
150 *arb_map_config = thrd_to_arb_map_sku6; 150 *arb_map_config = thrd_to_arb_map_sku6;
151 break; 151 break;
152 default: 152 default:
153 pr_err("QAT: The configuration doesn't match any SKU"); 153 dev_err(&GET_DEV(accel_dev),
154 "The configuration doesn't match any SKU");
154 *arb_map_config = NULL; 155 *arb_map_config = NULL;
155 } 156 }
156} 157}
diff --git a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.h b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.h
index 01e0be21e93a..25269a9f24a2 100644
--- a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.h
+++ b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.h
@@ -73,11 +73,11 @@
73/* Error detection and correction */ 73/* Error detection and correction */
74#define ADF_DH895XCC_AE_CTX_ENABLES(i) (i * 0x1000 + 0x20818) 74#define ADF_DH895XCC_AE_CTX_ENABLES(i) (i * 0x1000 + 0x20818)
75#define ADF_DH895XCC_AE_MISC_CONTROL(i) (i * 0x1000 + 0x20960) 75#define ADF_DH895XCC_AE_MISC_CONTROL(i) (i * 0x1000 + 0x20960)
76#define ADF_DH895XCC_ENABLE_AE_ECC_ERR (1 << 28) 76#define ADF_DH895XCC_ENABLE_AE_ECC_ERR BIT(28)
77#define ADF_DH895XCC_ENABLE_AE_ECC_PARITY_CORR (1 << 24 | 1 << 12) 77#define ADF_DH895XCC_ENABLE_AE_ECC_PARITY_CORR (BIT(24) | BIT(12))
78#define ADF_DH895XCC_UERRSSMSH(i) (i * 0x4000 + 0x18) 78#define ADF_DH895XCC_UERRSSMSH(i) (i * 0x4000 + 0x18)
79#define ADF_DH895XCC_CERRSSMSH(i) (i * 0x4000 + 0x10) 79#define ADF_DH895XCC_CERRSSMSH(i) (i * 0x4000 + 0x10)
80#define ADF_DH895XCC_ERRSSMSH_EN (1 << 3) 80#define ADF_DH895XCC_ERRSSMSH_EN BIT(3)
81 81
82/* Admin Messages Registers */ 82/* Admin Messages Registers */
83#define ADF_DH895XCC_ADMINMSGUR_OFFSET (0x3A000 + 0x574) 83#define ADF_DH895XCC_ADMINMSGUR_OFFSET (0x3A000 + 0x574)
diff --git a/drivers/crypto/qat/qat_dh895xcc/adf_drv.c b/drivers/crypto/qat/qat_dh895xcc/adf_drv.c
index 8ffdb95c9804..9decea2779c6 100644
--- a/drivers/crypto/qat/qat_dh895xcc/adf_drv.c
+++ b/drivers/crypto/qat/qat_dh895xcc/adf_drv.c
@@ -236,7 +236,7 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
236 } 236 }
237 237
238 accel_dev = kzalloc_node(sizeof(*accel_dev), GFP_KERNEL, 238 accel_dev = kzalloc_node(sizeof(*accel_dev), GFP_KERNEL,
239 dev_to_node(&pdev->dev)); 239 dev_to_node(&pdev->dev));
240 if (!accel_dev) 240 if (!accel_dev)
241 return -ENOMEM; 241 return -ENOMEM;
242 242
@@ -379,7 +379,7 @@ out_err:
379 return ret; 379 return ret;
380} 380}
381 381
382static void __exit adf_remove(struct pci_dev *pdev) 382static void adf_remove(struct pci_dev *pdev)
383{ 383{
384 struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev); 384 struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev);
385 385
diff --git a/drivers/crypto/qat/qat_dh895xcc/adf_isr.c b/drivers/crypto/qat/qat_dh895xcc/adf_isr.c
index fe8f89697ad8..0d03c109c2d3 100644
--- a/drivers/crypto/qat/qat_dh895xcc/adf_isr.c
+++ b/drivers/crypto/qat/qat_dh895xcc/adf_isr.c
@@ -73,7 +73,7 @@ static int adf_enable_msix(struct adf_accel_dev *accel_dev)
73 if (pci_enable_msix_exact(pci_dev_info->pci_dev, 73 if (pci_enable_msix_exact(pci_dev_info->pci_dev,
74 pci_dev_info->msix_entries.entries, 74 pci_dev_info->msix_entries.entries,
75 msix_num_entries)) { 75 msix_num_entries)) {
76 pr_err("QAT: Failed to enable MSIX IRQ\n"); 76 dev_err(&GET_DEV(accel_dev), "Failed to enable MSIX IRQ\n");
77 return -EFAULT; 77 return -EFAULT;
78 } 78 }
79 return 0; 79 return 0;
@@ -97,7 +97,8 @@ static irqreturn_t adf_msix_isr_ae(int irq, void *dev_ptr)
97{ 97{
98 struct adf_accel_dev *accel_dev = dev_ptr; 98 struct adf_accel_dev *accel_dev = dev_ptr;
99 99
100 pr_info("QAT: qat_dev%d spurious AE interrupt\n", accel_dev->accel_id); 100 dev_info(&GET_DEV(accel_dev), "qat_dev%d spurious AE interrupt\n",
101 accel_dev->accel_id);
101 return IRQ_HANDLED; 102 return IRQ_HANDLED;
102} 103}
103 104
@@ -121,8 +122,9 @@ static int adf_request_irqs(struct adf_accel_dev *accel_dev)
121 ret = request_irq(msixe[i].vector, 122 ret = request_irq(msixe[i].vector,
122 adf_msix_isr_bundle, 0, name, bank); 123 adf_msix_isr_bundle, 0, name, bank);
123 if (ret) { 124 if (ret) {
124 pr_err("QAT: failed to enable irq %d for %s\n", 125 dev_err(&GET_DEV(accel_dev),
125 msixe[i].vector, name); 126 "failed to enable irq %d for %s\n",
127 msixe[i].vector, name);
126 return ret; 128 return ret;
127 } 129 }
128 130
@@ -136,8 +138,9 @@ static int adf_request_irqs(struct adf_accel_dev *accel_dev)
136 "qat%d-ae-cluster", accel_dev->accel_id); 138 "qat%d-ae-cluster", accel_dev->accel_id);
137 ret = request_irq(msixe[i].vector, adf_msix_isr_ae, 0, name, accel_dev); 139 ret = request_irq(msixe[i].vector, adf_msix_isr_ae, 0, name, accel_dev);
138 if (ret) { 140 if (ret) {
139 pr_err("QAT: failed to enable irq %d, for %s\n", 141 dev_err(&GET_DEV(accel_dev),
140 msixe[i].vector, name); 142 "failed to enable irq %d, for %s\n",
143 msixe[i].vector, name);
141 return ret; 144 return ret;
142 } 145 }
143 return ret; 146 return ret;
diff --git a/drivers/crypto/sahara.c b/drivers/crypto/sahara.c
index 290a7f0a681f..6be377f6b9e7 100644
--- a/drivers/crypto/sahara.c
+++ b/drivers/crypto/sahara.c
@@ -479,6 +479,7 @@ static int sahara_hw_descriptor_create(struct sahara_dev *dev)
479 struct scatterlist *sg; 479 struct scatterlist *sg;
480 int ret; 480 int ret;
481 int i, j; 481 int i, j;
482 int idx = 0;
482 483
483 /* Copy new key if necessary */ 484 /* Copy new key if necessary */
484 if (ctx->flags & FLAGS_NEW_KEY) { 485 if (ctx->flags & FLAGS_NEW_KEY) {
@@ -486,17 +487,20 @@ static int sahara_hw_descriptor_create(struct sahara_dev *dev)
486 ctx->flags &= ~FLAGS_NEW_KEY; 487 ctx->flags &= ~FLAGS_NEW_KEY;
487 488
488 if (dev->flags & FLAGS_CBC) { 489 if (dev->flags & FLAGS_CBC) {
489 dev->hw_desc[0]->len1 = AES_BLOCK_SIZE; 490 dev->hw_desc[idx]->len1 = AES_BLOCK_SIZE;
490 dev->hw_desc[0]->p1 = dev->iv_phys_base; 491 dev->hw_desc[idx]->p1 = dev->iv_phys_base;
491 } else { 492 } else {
492 dev->hw_desc[0]->len1 = 0; 493 dev->hw_desc[idx]->len1 = 0;
493 dev->hw_desc[0]->p1 = 0; 494 dev->hw_desc[idx]->p1 = 0;
494 } 495 }
495 dev->hw_desc[0]->len2 = ctx->keylen; 496 dev->hw_desc[idx]->len2 = ctx->keylen;
496 dev->hw_desc[0]->p2 = dev->key_phys_base; 497 dev->hw_desc[idx]->p2 = dev->key_phys_base;
497 dev->hw_desc[0]->next = dev->hw_phys_desc[1]; 498 dev->hw_desc[idx]->next = dev->hw_phys_desc[1];
499
500 dev->hw_desc[idx]->hdr = sahara_aes_key_hdr(dev);
501
502 idx++;
498 } 503 }
499 dev->hw_desc[0]->hdr = sahara_aes_key_hdr(dev);
500 504
501 dev->nb_in_sg = sahara_sg_length(dev->in_sg, dev->total); 505 dev->nb_in_sg = sahara_sg_length(dev->in_sg, dev->total);
502 dev->nb_out_sg = sahara_sg_length(dev->out_sg, dev->total); 506 dev->nb_out_sg = sahara_sg_length(dev->out_sg, dev->total);
@@ -520,7 +524,7 @@ static int sahara_hw_descriptor_create(struct sahara_dev *dev)
520 } 524 }
521 525
522 /* Create input links */ 526 /* Create input links */
523 dev->hw_desc[1]->p1 = dev->hw_phys_link[0]; 527 dev->hw_desc[idx]->p1 = dev->hw_phys_link[0];
524 sg = dev->in_sg; 528 sg = dev->in_sg;
525 for (i = 0; i < dev->nb_in_sg; i++) { 529 for (i = 0; i < dev->nb_in_sg; i++) {
526 dev->hw_link[i]->len = sg->length; 530 dev->hw_link[i]->len = sg->length;
@@ -534,7 +538,7 @@ static int sahara_hw_descriptor_create(struct sahara_dev *dev)
534 } 538 }
535 539
536 /* Create output links */ 540 /* Create output links */
537 dev->hw_desc[1]->p2 = dev->hw_phys_link[i]; 541 dev->hw_desc[idx]->p2 = dev->hw_phys_link[i];
538 sg = dev->out_sg; 542 sg = dev->out_sg;
539 for (j = i; j < dev->nb_out_sg + i; j++) { 543 for (j = i; j < dev->nb_out_sg + i; j++) {
540 dev->hw_link[j]->len = sg->length; 544 dev->hw_link[j]->len = sg->length;
@@ -548,10 +552,10 @@ static int sahara_hw_descriptor_create(struct sahara_dev *dev)
548 } 552 }
549 553
550 /* Fill remaining fields of hw_desc[1] */ 554 /* Fill remaining fields of hw_desc[1] */
551 dev->hw_desc[1]->hdr = sahara_aes_data_link_hdr(dev); 555 dev->hw_desc[idx]->hdr = sahara_aes_data_link_hdr(dev);
552 dev->hw_desc[1]->len1 = dev->total; 556 dev->hw_desc[idx]->len1 = dev->total;
553 dev->hw_desc[1]->len2 = dev->total; 557 dev->hw_desc[idx]->len2 = dev->total;
554 dev->hw_desc[1]->next = 0; 558 dev->hw_desc[idx]->next = 0;
555 559
556 sahara_dump_descriptors(dev); 560 sahara_dump_descriptors(dev);
557 sahara_dump_links(dev); 561 sahara_dump_links(dev);
@@ -576,6 +580,7 @@ static int sahara_aes_process(struct ablkcipher_request *req)
576 struct sahara_ctx *ctx; 580 struct sahara_ctx *ctx;
577 struct sahara_aes_reqctx *rctx; 581 struct sahara_aes_reqctx *rctx;
578 int ret; 582 int ret;
583 unsigned long timeout;
579 584
580 /* Request is ready to be dispatched by the device */ 585 /* Request is ready to be dispatched by the device */
581 dev_dbg(dev->device, 586 dev_dbg(dev->device,
@@ -601,10 +606,12 @@ static int sahara_aes_process(struct ablkcipher_request *req)
601 reinit_completion(&dev->dma_completion); 606 reinit_completion(&dev->dma_completion);
602 607
603 ret = sahara_hw_descriptor_create(dev); 608 ret = sahara_hw_descriptor_create(dev);
609 if (ret)
610 return -EINVAL;
604 611
605 ret = wait_for_completion_timeout(&dev->dma_completion, 612 timeout = wait_for_completion_timeout(&dev->dma_completion,
606 msecs_to_jiffies(SAHARA_TIMEOUT_MS)); 613 msecs_to_jiffies(SAHARA_TIMEOUT_MS));
607 if (!ret) { 614 if (!timeout) {
608 dev_err(dev->device, "AES timeout\n"); 615 dev_err(dev->device, "AES timeout\n");
609 return -ETIMEDOUT; 616 return -ETIMEDOUT;
610 } 617 }
@@ -1044,7 +1051,8 @@ static int sahara_sha_process(struct ahash_request *req)
1044{ 1051{
1045 struct sahara_dev *dev = dev_ptr; 1052 struct sahara_dev *dev = dev_ptr;
1046 struct sahara_sha_reqctx *rctx = ahash_request_ctx(req); 1053 struct sahara_sha_reqctx *rctx = ahash_request_ctx(req);
1047 int ret = -EINPROGRESS; 1054 int ret;
1055 unsigned long timeout;
1048 1056
1049 ret = sahara_sha_prepare_request(req); 1057 ret = sahara_sha_prepare_request(req);
1050 if (!ret) 1058 if (!ret)
@@ -1070,9 +1078,9 @@ static int sahara_sha_process(struct ahash_request *req)
1070 1078
1071 sahara_write(dev, dev->hw_phys_desc[0], SAHARA_REG_DAR); 1079 sahara_write(dev, dev->hw_phys_desc[0], SAHARA_REG_DAR);
1072 1080
1073 ret = wait_for_completion_timeout(&dev->dma_completion, 1081 timeout = wait_for_completion_timeout(&dev->dma_completion,
1074 msecs_to_jiffies(SAHARA_TIMEOUT_MS)); 1082 msecs_to_jiffies(SAHARA_TIMEOUT_MS));
1075 if (!ret) { 1083 if (!timeout) {
1076 dev_err(dev->device, "SHA timeout\n"); 1084 dev_err(dev->device, "SHA timeout\n");
1077 return -ETIMEDOUT; 1085 return -ETIMEDOUT;
1078 } 1086 }
@@ -1092,15 +1100,20 @@ static int sahara_queue_manage(void *data)
1092{ 1100{
1093 struct sahara_dev *dev = (struct sahara_dev *)data; 1101 struct sahara_dev *dev = (struct sahara_dev *)data;
1094 struct crypto_async_request *async_req; 1102 struct crypto_async_request *async_req;
1103 struct crypto_async_request *backlog;
1095 int ret = 0; 1104 int ret = 0;
1096 1105
1097 do { 1106 do {
1098 __set_current_state(TASK_INTERRUPTIBLE); 1107 __set_current_state(TASK_INTERRUPTIBLE);
1099 1108
1100 mutex_lock(&dev->queue_mutex); 1109 mutex_lock(&dev->queue_mutex);
1110 backlog = crypto_get_backlog(&dev->queue);
1101 async_req = crypto_dequeue_request(&dev->queue); 1111 async_req = crypto_dequeue_request(&dev->queue);
1102 mutex_unlock(&dev->queue_mutex); 1112 mutex_unlock(&dev->queue_mutex);
1103 1113
1114 if (backlog)
1115 backlog->complete(backlog, -EINPROGRESS);
1116
1104 if (async_req) { 1117 if (async_req) {
1105 if (crypto_tfm_alg_type(async_req->tfm) == 1118 if (crypto_tfm_alg_type(async_req->tfm) ==
1106 CRYPTO_ALG_TYPE_AHASH) { 1119 CRYPTO_ALG_TYPE_AHASH) {
diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c
index ebbae8d3ce0d..857414afa29a 100644
--- a/drivers/crypto/talitos.c
+++ b/drivers/crypto/talitos.c
@@ -637,8 +637,6 @@ static void talitos_unregister_rng(struct device *dev)
637#define TALITOS_MAX_KEY_SIZE 96 637#define TALITOS_MAX_KEY_SIZE 96
638#define TALITOS_MAX_IV_LENGTH 16 /* max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */ 638#define TALITOS_MAX_IV_LENGTH 16 /* max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */
639 639
640#define MD5_BLOCK_SIZE 64
641
642struct talitos_ctx { 640struct talitos_ctx {
643 struct device *dev; 641 struct device *dev;
644 int ch; 642 int ch;
@@ -2195,7 +2193,7 @@ static struct talitos_alg_template driver_algs[] = {
2195 .halg.base = { 2193 .halg.base = {
2196 .cra_name = "md5", 2194 .cra_name = "md5",
2197 .cra_driver_name = "md5-talitos", 2195 .cra_driver_name = "md5-talitos",
2198 .cra_blocksize = MD5_BLOCK_SIZE, 2196 .cra_blocksize = MD5_HMAC_BLOCK_SIZE,
2199 .cra_flags = CRYPTO_ALG_TYPE_AHASH | 2197 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2200 CRYPTO_ALG_ASYNC, 2198 CRYPTO_ALG_ASYNC,
2201 } 2199 }
@@ -2285,7 +2283,7 @@ static struct talitos_alg_template driver_algs[] = {
2285 .halg.base = { 2283 .halg.base = {
2286 .cra_name = "hmac(md5)", 2284 .cra_name = "hmac(md5)",
2287 .cra_driver_name = "hmac-md5-talitos", 2285 .cra_driver_name = "hmac-md5-talitos",
2288 .cra_blocksize = MD5_BLOCK_SIZE, 2286 .cra_blocksize = MD5_HMAC_BLOCK_SIZE,
2289 .cra_flags = CRYPTO_ALG_TYPE_AHASH | 2287 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2290 CRYPTO_ALG_ASYNC, 2288 CRYPTO_ALG_ASYNC,
2291 } 2289 }
@@ -2706,20 +2704,16 @@ static int talitos_probe(struct platform_device *ofdev)
2706 goto err_out; 2704 goto err_out;
2707 } 2705 }
2708 2706
2707 priv->fifo_len = roundup_pow_of_two(priv->chfifo_len);
2708
2709 for (i = 0; i < priv->num_channels; i++) { 2709 for (i = 0; i < priv->num_channels; i++) {
2710 priv->chan[i].reg = priv->reg + TALITOS_CH_STRIDE * (i + 1); 2710 priv->chan[i].reg = priv->reg + TALITOS_CH_STRIDE * (i + 1);
2711 if (!priv->irq[1] || !(i & 1)) 2711 if (!priv->irq[1] || !(i & 1))
2712 priv->chan[i].reg += TALITOS_CH_BASE_OFFSET; 2712 priv->chan[i].reg += TALITOS_CH_BASE_OFFSET;
2713 }
2714 2713
2715 for (i = 0; i < priv->num_channels; i++) {
2716 spin_lock_init(&priv->chan[i].head_lock); 2714 spin_lock_init(&priv->chan[i].head_lock);
2717 spin_lock_init(&priv->chan[i].tail_lock); 2715 spin_lock_init(&priv->chan[i].tail_lock);
2718 }
2719 2716
2720 priv->fifo_len = roundup_pow_of_two(priv->chfifo_len);
2721
2722 for (i = 0; i < priv->num_channels; i++) {
2723 priv->chan[i].fifo = kzalloc(sizeof(struct talitos_request) * 2717 priv->chan[i].fifo = kzalloc(sizeof(struct talitos_request) *
2724 priv->fifo_len, GFP_KERNEL); 2718 priv->fifo_len, GFP_KERNEL);
2725 if (!priv->chan[i].fifo) { 2719 if (!priv->chan[i].fifo) {
@@ -2727,11 +2721,10 @@ static int talitos_probe(struct platform_device *ofdev)
2727 err = -ENOMEM; 2721 err = -ENOMEM;
2728 goto err_out; 2722 goto err_out;
2729 } 2723 }
2730 }
2731 2724
2732 for (i = 0; i < priv->num_channels; i++)
2733 atomic_set(&priv->chan[i].submit_count, 2725 atomic_set(&priv->chan[i].submit_count,
2734 -(priv->chfifo_len - 1)); 2726 -(priv->chfifo_len - 1));
2727 }
2735 2728
2736 dma_set_mask(dev, DMA_BIT_MASK(36)); 2729 dma_set_mask(dev, DMA_BIT_MASK(36));
2737 2730
diff --git a/drivers/crypto/ux500/hash/hash_core.c b/drivers/crypto/ux500/hash/hash_core.c
index 187a8fd7eee7..5f5f360628fc 100644
--- a/drivers/crypto/ux500/hash/hash_core.c
+++ b/drivers/crypto/ux500/hash/hash_core.c
@@ -184,7 +184,7 @@ static int hash_set_dma_transfer(struct hash_ctx *ctx, struct scatterlist *sg,
184 direction, DMA_CTRL_ACK | DMA_PREP_INTERRUPT); 184 direction, DMA_CTRL_ACK | DMA_PREP_INTERRUPT);
185 if (!desc) { 185 if (!desc) {
186 dev_err(ctx->device->dev, 186 dev_err(ctx->device->dev,
187 "%s: device_prep_slave_sg() failed!\n", __func__); 187 "%s: dmaengine_prep_slave_sg() failed!\n", __func__);
188 return -EFAULT; 188 return -EFAULT;
189 } 189 }
190 190
diff --git a/drivers/crypto/vmx/Kconfig b/drivers/crypto/vmx/Kconfig
new file mode 100644
index 000000000000..771babf16aa0
--- /dev/null
+++ b/drivers/crypto/vmx/Kconfig
@@ -0,0 +1,8 @@
1config CRYPTO_DEV_VMX_ENCRYPT
2 tristate "Encryption acceleration support on P8 CPU"
3 depends on PPC64 && CRYPTO_DEV_VMX
4 default y
5 help
6 Support for VMX cryptographic acceleration instructions on Power8 CPU.
7 This module supports acceleration for AES and GHASH in hardware. If you
8 choose 'M' here, this module will be called vmx-crypto.
diff --git a/drivers/crypto/vmx/Makefile b/drivers/crypto/vmx/Makefile
new file mode 100644
index 000000000000..c699c6e6c82e
--- /dev/null
+++ b/drivers/crypto/vmx/Makefile
@@ -0,0 +1,19 @@
1obj-$(CONFIG_CRYPTO_DEV_VMX_ENCRYPT) += vmx-crypto.o
2vmx-crypto-objs := vmx.o aesp8-ppc.o ghashp8-ppc.o aes.o aes_cbc.o aes_ctr.o ghash.o
3
4ifeq ($(CONFIG_CPU_LITTLE_ENDIAN),y)
5TARGET := linux-ppc64le
6else
7TARGET := linux-pcc64
8endif
9
10quiet_cmd_perl = PERL $@
11 cmd_perl = $(PERL) $(<) $(TARGET) > $(@)
12
13$(src)/aesp8-ppc.S: $(src)/aesp8-ppc.pl
14 $(call cmd,perl)
15
16$(src)/ghashp8-ppc.S: $(src)/ghashp8-ppc.pl
17 $(call cmd,perl)
18
19.PRECIOUS: $(obj)/aesp8-ppc.S $(obj)/ghashp8-ppc.S
diff --git a/drivers/crypto/vmx/aes.c b/drivers/crypto/vmx/aes.c
new file mode 100644
index 000000000000..ab300ea19434
--- /dev/null
+++ b/drivers/crypto/vmx/aes.c
@@ -0,0 +1,139 @@
1/**
2 * AES routines supporting VMX instructions on the Power 8
3 *
4 * Copyright (C) 2015 International Business Machines Inc.
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; version 2 only.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
18 *
19 * Author: Marcelo Henrique Cerri <mhcerri@br.ibm.com>
20 */
21
22#include <linux/types.h>
23#include <linux/err.h>
24#include <linux/crypto.h>
25#include <linux/delay.h>
26#include <linux/hardirq.h>
27#include <asm/switch_to.h>
28#include <crypto/aes.h>
29
30#include "aesp8-ppc.h"
31
32struct p8_aes_ctx {
33 struct crypto_cipher *fallback;
34 struct aes_key enc_key;
35 struct aes_key dec_key;
36};
37
38static int p8_aes_init(struct crypto_tfm *tfm)
39{
40 const char *alg;
41 struct crypto_cipher *fallback;
42 struct p8_aes_ctx *ctx = crypto_tfm_ctx(tfm);
43
44 if (!(alg = crypto_tfm_alg_name(tfm))) {
45 printk(KERN_ERR "Failed to get algorithm name.\n");
46 return -ENOENT;
47 }
48
49 fallback = crypto_alloc_cipher(alg, 0 ,CRYPTO_ALG_NEED_FALLBACK);
50 if (IS_ERR(fallback)) {
51 printk(KERN_ERR "Failed to allocate transformation for '%s': %ld\n",
52 alg, PTR_ERR(fallback));
53 return PTR_ERR(fallback);
54 }
55 printk(KERN_INFO "Using '%s' as fallback implementation.\n",
56 crypto_tfm_alg_driver_name((struct crypto_tfm *) fallback));
57
58 crypto_cipher_set_flags(fallback,
59 crypto_cipher_get_flags((struct crypto_cipher *) tfm));
60 ctx->fallback = fallback;
61
62 return 0;
63}
64
65static void p8_aes_exit(struct crypto_tfm *tfm)
66{
67 struct p8_aes_ctx *ctx = crypto_tfm_ctx(tfm);
68
69 if (ctx->fallback) {
70 crypto_free_cipher(ctx->fallback);
71 ctx->fallback = NULL;
72 }
73}
74
75static int p8_aes_setkey(struct crypto_tfm *tfm, const u8 *key,
76 unsigned int keylen)
77{
78 int ret;
79 struct p8_aes_ctx *ctx = crypto_tfm_ctx(tfm);
80
81 pagefault_disable();
82 enable_kernel_altivec();
83 ret = aes_p8_set_encrypt_key(key, keylen * 8, &ctx->enc_key);
84 ret += aes_p8_set_decrypt_key(key, keylen * 8, &ctx->dec_key);
85 pagefault_enable();
86
87 ret += crypto_cipher_setkey(ctx->fallback, key, keylen);
88 return ret;
89}
90
91static void p8_aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
92{
93 struct p8_aes_ctx *ctx = crypto_tfm_ctx(tfm);
94
95 if (in_interrupt()) {
96 crypto_cipher_encrypt_one(ctx->fallback, dst, src);
97 } else {
98 pagefault_disable();
99 enable_kernel_altivec();
100 aes_p8_encrypt(src, dst, &ctx->enc_key);
101 pagefault_enable();
102 }
103}
104
105static void p8_aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
106{
107 struct p8_aes_ctx *ctx = crypto_tfm_ctx(tfm);
108
109 if (in_interrupt()) {
110 crypto_cipher_decrypt_one(ctx->fallback, dst, src);
111 } else {
112 pagefault_disable();
113 enable_kernel_altivec();
114 aes_p8_decrypt(src, dst, &ctx->dec_key);
115 pagefault_enable();
116 }
117}
118
119struct crypto_alg p8_aes_alg = {
120 .cra_name = "aes",
121 .cra_driver_name = "p8_aes",
122 .cra_module = THIS_MODULE,
123 .cra_priority = 1000,
124 .cra_type = NULL,
125 .cra_flags = CRYPTO_ALG_TYPE_CIPHER | CRYPTO_ALG_NEED_FALLBACK,
126 .cra_alignmask = 0,
127 .cra_blocksize = AES_BLOCK_SIZE,
128 .cra_ctxsize = sizeof(struct p8_aes_ctx),
129 .cra_init = p8_aes_init,
130 .cra_exit = p8_aes_exit,
131 .cra_cipher = {
132 .cia_min_keysize = AES_MIN_KEY_SIZE,
133 .cia_max_keysize = AES_MAX_KEY_SIZE,
134 .cia_setkey = p8_aes_setkey,
135 .cia_encrypt = p8_aes_encrypt,
136 .cia_decrypt = p8_aes_decrypt,
137 },
138};
139
diff --git a/drivers/crypto/vmx/aes_cbc.c b/drivers/crypto/vmx/aes_cbc.c
new file mode 100644
index 000000000000..1a559b7dddb5
--- /dev/null
+++ b/drivers/crypto/vmx/aes_cbc.c
@@ -0,0 +1,184 @@
1/**
2 * AES CBC routines supporting VMX instructions on the Power 8
3 *
4 * Copyright (C) 2015 International Business Machines Inc.
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; version 2 only.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
18 *
19 * Author: Marcelo Henrique Cerri <mhcerri@br.ibm.com>
20 */
21
22#include <linux/types.h>
23#include <linux/err.h>
24#include <linux/crypto.h>
25#include <linux/delay.h>
26#include <linux/hardirq.h>
27#include <asm/switch_to.h>
28#include <crypto/aes.h>
29#include <crypto/scatterwalk.h>
30
31#include "aesp8-ppc.h"
32
33struct p8_aes_cbc_ctx {
34 struct crypto_blkcipher *fallback;
35 struct aes_key enc_key;
36 struct aes_key dec_key;
37};
38
39static int p8_aes_cbc_init(struct crypto_tfm *tfm)
40{
41 const char *alg;
42 struct crypto_blkcipher *fallback;
43 struct p8_aes_cbc_ctx *ctx = crypto_tfm_ctx(tfm);
44
45 if (!(alg = crypto_tfm_alg_name(tfm))) {
46 printk(KERN_ERR "Failed to get algorithm name.\n");
47 return -ENOENT;
48 }
49
50 fallback = crypto_alloc_blkcipher(alg, 0 ,CRYPTO_ALG_NEED_FALLBACK);
51 if (IS_ERR(fallback)) {
52 printk(KERN_ERR "Failed to allocate transformation for '%s': %ld\n",
53 alg, PTR_ERR(fallback));
54 return PTR_ERR(fallback);
55 }
56 printk(KERN_INFO "Using '%s' as fallback implementation.\n",
57 crypto_tfm_alg_driver_name((struct crypto_tfm *) fallback));
58
59 crypto_blkcipher_set_flags(fallback,
60 crypto_blkcipher_get_flags((struct crypto_blkcipher *) tfm));
61 ctx->fallback = fallback;
62
63 return 0;
64}
65
66static void p8_aes_cbc_exit(struct crypto_tfm *tfm)
67{
68 struct p8_aes_cbc_ctx *ctx = crypto_tfm_ctx(tfm);
69
70 if (ctx->fallback) {
71 crypto_free_blkcipher(ctx->fallback);
72 ctx->fallback = NULL;
73 }
74}
75
76static int p8_aes_cbc_setkey(struct crypto_tfm *tfm, const u8 *key,
77 unsigned int keylen)
78{
79 int ret;
80 struct p8_aes_cbc_ctx *ctx = crypto_tfm_ctx(tfm);
81
82 pagefault_disable();
83 enable_kernel_altivec();
84 ret = aes_p8_set_encrypt_key(key, keylen * 8, &ctx->enc_key);
85 ret += aes_p8_set_decrypt_key(key, keylen * 8, &ctx->dec_key);
86 pagefault_enable();
87
88 ret += crypto_blkcipher_setkey(ctx->fallback, key, keylen);
89 return ret;
90}
91
92static int p8_aes_cbc_encrypt(struct blkcipher_desc *desc,
93 struct scatterlist *dst, struct scatterlist *src,
94 unsigned int nbytes)
95{
96 int ret;
97 struct blkcipher_walk walk;
98 struct p8_aes_cbc_ctx *ctx = crypto_tfm_ctx(
99 crypto_blkcipher_tfm(desc->tfm));
100 struct blkcipher_desc fallback_desc = {
101 .tfm = ctx->fallback,
102 .info = desc->info,
103 .flags = desc->flags
104 };
105
106 if (in_interrupt()) {
107 ret = crypto_blkcipher_encrypt(&fallback_desc, dst, src, nbytes);
108 } else {
109 pagefault_disable();
110 enable_kernel_altivec();
111
112 blkcipher_walk_init(&walk, dst, src, nbytes);
113 ret = blkcipher_walk_virt(desc, &walk);
114 while ((nbytes = walk.nbytes)) {
115 aes_p8_cbc_encrypt(walk.src.virt.addr, walk.dst.virt.addr,
116 nbytes & AES_BLOCK_MASK, &ctx->enc_key, walk.iv, 1);
117 nbytes &= AES_BLOCK_SIZE - 1;
118 ret = blkcipher_walk_done(desc, &walk, nbytes);
119 }
120
121 pagefault_enable();
122 }
123
124 return ret;
125}
126
127static int p8_aes_cbc_decrypt(struct blkcipher_desc *desc,
128 struct scatterlist *dst, struct scatterlist *src,
129 unsigned int nbytes)
130{
131 int ret;
132 struct blkcipher_walk walk;
133 struct p8_aes_cbc_ctx *ctx = crypto_tfm_ctx(
134 crypto_blkcipher_tfm(desc->tfm));
135 struct blkcipher_desc fallback_desc = {
136 .tfm = ctx->fallback,
137 .info = desc->info,
138 .flags = desc->flags
139 };
140
141 if (in_interrupt()) {
142 ret = crypto_blkcipher_decrypt(&fallback_desc, dst, src, nbytes);
143 } else {
144 pagefault_disable();
145 enable_kernel_altivec();
146
147 blkcipher_walk_init(&walk, dst, src, nbytes);
148 ret = blkcipher_walk_virt(desc, &walk);
149 while ((nbytes = walk.nbytes)) {
150 aes_p8_cbc_encrypt(walk.src.virt.addr, walk.dst.virt.addr,
151 nbytes & AES_BLOCK_MASK, &ctx->dec_key, walk.iv, 0);
152 nbytes &= AES_BLOCK_SIZE - 1;
153 ret = blkcipher_walk_done(desc, &walk, nbytes);
154 }
155
156 pagefault_enable();
157 }
158
159 return ret;
160}
161
162
163struct crypto_alg p8_aes_cbc_alg = {
164 .cra_name = "cbc(aes)",
165 .cra_driver_name = "p8_aes_cbc",
166 .cra_module = THIS_MODULE,
167 .cra_priority = 1000,
168 .cra_type = &crypto_blkcipher_type,
169 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | CRYPTO_ALG_NEED_FALLBACK,
170 .cra_alignmask = 0,
171 .cra_blocksize = AES_BLOCK_SIZE,
172 .cra_ctxsize = sizeof(struct p8_aes_cbc_ctx),
173 .cra_init = p8_aes_cbc_init,
174 .cra_exit = p8_aes_cbc_exit,
175 .cra_blkcipher = {
176 .ivsize = 0,
177 .min_keysize = AES_MIN_KEY_SIZE,
178 .max_keysize = AES_MAX_KEY_SIZE,
179 .setkey = p8_aes_cbc_setkey,
180 .encrypt = p8_aes_cbc_encrypt,
181 .decrypt = p8_aes_cbc_decrypt,
182 },
183};
184
diff --git a/drivers/crypto/vmx/aes_ctr.c b/drivers/crypto/vmx/aes_ctr.c
new file mode 100644
index 000000000000..96dbee4bf4a6
--- /dev/null
+++ b/drivers/crypto/vmx/aes_ctr.c
@@ -0,0 +1,167 @@
1/**
2 * AES CTR routines supporting VMX instructions on the Power 8
3 *
4 * Copyright (C) 2015 International Business Machines Inc.
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; version 2 only.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
18 *
19 * Author: Marcelo Henrique Cerri <mhcerri@br.ibm.com>
20 */
21
22#include <linux/types.h>
23#include <linux/err.h>
24#include <linux/crypto.h>
25#include <linux/delay.h>
26#include <linux/hardirq.h>
27#include <asm/switch_to.h>
28#include <crypto/aes.h>
29#include <crypto/scatterwalk.h>
30#include "aesp8-ppc.h"
31
32struct p8_aes_ctr_ctx {
33 struct crypto_blkcipher *fallback;
34 struct aes_key enc_key;
35};
36
37static int p8_aes_ctr_init(struct crypto_tfm *tfm)
38{
39 const char *alg;
40 struct crypto_blkcipher *fallback;
41 struct p8_aes_ctr_ctx *ctx = crypto_tfm_ctx(tfm);
42
43 if (!(alg = crypto_tfm_alg_name(tfm))) {
44 printk(KERN_ERR "Failed to get algorithm name.\n");
45 return -ENOENT;
46 }
47
48 fallback = crypto_alloc_blkcipher(alg, 0 ,CRYPTO_ALG_NEED_FALLBACK);
49 if (IS_ERR(fallback)) {
50 printk(KERN_ERR "Failed to allocate transformation for '%s': %ld\n",
51 alg, PTR_ERR(fallback));
52 return PTR_ERR(fallback);
53 }
54 printk(KERN_INFO "Using '%s' as fallback implementation.\n",
55 crypto_tfm_alg_driver_name((struct crypto_tfm *) fallback));
56
57 crypto_blkcipher_set_flags(fallback,
58 crypto_blkcipher_get_flags((struct crypto_blkcipher *) tfm));
59 ctx->fallback = fallback;
60
61 return 0;
62}
63
64static void p8_aes_ctr_exit(struct crypto_tfm *tfm)
65{
66 struct p8_aes_ctr_ctx *ctx = crypto_tfm_ctx(tfm);
67
68 if (ctx->fallback) {
69 crypto_free_blkcipher(ctx->fallback);
70 ctx->fallback = NULL;
71 }
72}
73
74static int p8_aes_ctr_setkey(struct crypto_tfm *tfm, const u8 *key,
75 unsigned int keylen)
76{
77 int ret;
78 struct p8_aes_ctr_ctx *ctx = crypto_tfm_ctx(tfm);
79
80 pagefault_disable();
81 enable_kernel_altivec();
82 ret = aes_p8_set_encrypt_key(key, keylen * 8, &ctx->enc_key);
83 pagefault_enable();
84
85 ret += crypto_blkcipher_setkey(ctx->fallback, key, keylen);
86 return ret;
87}
88
89static void p8_aes_ctr_final(struct p8_aes_ctr_ctx *ctx,
90 struct blkcipher_walk *walk)
91{
92 u8 *ctrblk = walk->iv;
93 u8 keystream[AES_BLOCK_SIZE];
94 u8 *src = walk->src.virt.addr;
95 u8 *dst = walk->dst.virt.addr;
96 unsigned int nbytes = walk->nbytes;
97
98 pagefault_disable();
99 enable_kernel_altivec();
100 aes_p8_encrypt(ctrblk, keystream, &ctx->enc_key);
101 pagefault_enable();
102
103 crypto_xor(keystream, src, nbytes);
104 memcpy(dst, keystream, nbytes);
105 crypto_inc(ctrblk, AES_BLOCK_SIZE);
106}
107
108static int p8_aes_ctr_crypt(struct blkcipher_desc *desc,
109 struct scatterlist *dst, struct scatterlist *src,
110 unsigned int nbytes)
111{
112 int ret;
113 struct blkcipher_walk walk;
114 struct p8_aes_ctr_ctx *ctx = crypto_tfm_ctx(
115 crypto_blkcipher_tfm(desc->tfm));
116 struct blkcipher_desc fallback_desc = {
117 .tfm = ctx->fallback,
118 .info = desc->info,
119 .flags = desc->flags
120 };
121
122 if (in_interrupt()) {
123 ret = crypto_blkcipher_encrypt(&fallback_desc, dst, src, nbytes);
124 } else {
125 blkcipher_walk_init(&walk, dst, src, nbytes);
126 ret = blkcipher_walk_virt_block(desc, &walk, AES_BLOCK_SIZE);
127 while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) {
128 pagefault_disable();
129 enable_kernel_altivec();
130 aes_p8_ctr32_encrypt_blocks(walk.src.virt.addr, walk.dst.virt.addr,
131 (nbytes & AES_BLOCK_MASK)/AES_BLOCK_SIZE, &ctx->enc_key, walk.iv);
132 pagefault_enable();
133
134 crypto_inc(walk.iv, AES_BLOCK_SIZE);
135 nbytes &= AES_BLOCK_SIZE - 1;
136 ret = blkcipher_walk_done(desc, &walk, nbytes);
137 }
138 if (walk.nbytes) {
139 p8_aes_ctr_final(ctx, &walk);
140 ret = blkcipher_walk_done(desc, &walk, 0);
141 }
142 }
143
144 return ret;
145}
146
147struct crypto_alg p8_aes_ctr_alg = {
148 .cra_name = "ctr(aes)",
149 .cra_driver_name = "p8_aes_ctr",
150 .cra_module = THIS_MODULE,
151 .cra_priority = 1000,
152 .cra_type = &crypto_blkcipher_type,
153 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | CRYPTO_ALG_NEED_FALLBACK,
154 .cra_alignmask = 0,
155 .cra_blocksize = 1,
156 .cra_ctxsize = sizeof(struct p8_aes_ctr_ctx),
157 .cra_init = p8_aes_ctr_init,
158 .cra_exit = p8_aes_ctr_exit,
159 .cra_blkcipher = {
160 .ivsize = 0,
161 .min_keysize = AES_MIN_KEY_SIZE,
162 .max_keysize = AES_MAX_KEY_SIZE,
163 .setkey = p8_aes_ctr_setkey,
164 .encrypt = p8_aes_ctr_crypt,
165 .decrypt = p8_aes_ctr_crypt,
166 },
167};
diff --git a/drivers/crypto/vmx/aesp8-ppc.h b/drivers/crypto/vmx/aesp8-ppc.h
new file mode 100644
index 000000000000..e963945a83e1
--- /dev/null
+++ b/drivers/crypto/vmx/aesp8-ppc.h
@@ -0,0 +1,20 @@
1#include <linux/types.h>
2#include <crypto/aes.h>
3
4#define AES_BLOCK_MASK (~(AES_BLOCK_SIZE-1))
5
6struct aes_key {
7 u8 key[AES_MAX_KEYLENGTH];
8 int rounds;
9};
10
11int aes_p8_set_encrypt_key(const u8 *userKey, const int bits,
12 struct aes_key *key);
13int aes_p8_set_decrypt_key(const u8 *userKey, const int bits,
14 struct aes_key *key);
15void aes_p8_encrypt(const u8 *in, u8 *out, const struct aes_key *key);
16void aes_p8_decrypt(const u8 *in, u8 *out,const struct aes_key *key);
17void aes_p8_cbc_encrypt(const u8 *in, u8 *out, size_t len,
18 const struct aes_key *key, u8 *iv, const int enc);
19void aes_p8_ctr32_encrypt_blocks(const u8 *in, u8 *out,
20 size_t len, const struct aes_key *key, const u8 *iv);
diff --git a/drivers/crypto/vmx/aesp8-ppc.pl b/drivers/crypto/vmx/aesp8-ppc.pl
new file mode 100644
index 000000000000..6c5c20c6108e
--- /dev/null
+++ b/drivers/crypto/vmx/aesp8-ppc.pl
@@ -0,0 +1,1930 @@
1#!/usr/bin/env perl
2#
3# ====================================================================
4# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
5# project. The module is, however, dual licensed under OpenSSL and
6# CRYPTOGAMS licenses depending on where you obtain it. For further
7# details see http://www.openssl.org/~appro/cryptogams/.
8# ====================================================================
9#
10# This module implements support for AES instructions as per PowerISA
11# specification version 2.07, first implemented by POWER8 processor.
12# The module is endian-agnostic in sense that it supports both big-
13# and little-endian cases. Data alignment in parallelizable modes is
14# handled with VSX loads and stores, which implies MSR.VSX flag being
15# set. It should also be noted that ISA specification doesn't prohibit
16# alignment exceptions for these instructions on page boundaries.
17# Initially alignment was handled in pure AltiVec/VMX way [when data
18# is aligned programmatically, which in turn guarantees exception-
19# free execution], but it turned to hamper performance when vcipher
20# instructions are interleaved. It's reckoned that eventual
21# misalignment penalties at page boundaries are in average lower
22# than additional overhead in pure AltiVec approach.
23
24$flavour = shift;
25
26if ($flavour =~ /64/) {
27 $SIZE_T =8;
28 $LRSAVE =2*$SIZE_T;
29 $STU ="stdu";
30 $POP ="ld";
31 $PUSH ="std";
32 $UCMP ="cmpld";
33 $SHL ="sldi";
34} elsif ($flavour =~ /32/) {
35 $SIZE_T =4;
36 $LRSAVE =$SIZE_T;
37 $STU ="stwu";
38 $POP ="lwz";
39 $PUSH ="stw";
40 $UCMP ="cmplw";
41 $SHL ="slwi";
42} else { die "nonsense $flavour"; }
43
44$LITTLE_ENDIAN = ($flavour=~/le$/) ? $SIZE_T : 0;
45
46$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
47( $xlate="${dir}ppc-xlate.pl" and -f $xlate ) or
48( $xlate="${dir}../../perlasm/ppc-xlate.pl" and -f $xlate) or
49die "can't locate ppc-xlate.pl";
50
51open STDOUT,"| $^X $xlate $flavour ".shift || die "can't call $xlate: $!";
52
53$FRAME=8*$SIZE_T;
54$prefix="aes_p8";
55
56$sp="r1";
57$vrsave="r12";
58
59#########################################################################
60{{{ # Key setup procedures #
61my ($inp,$bits,$out,$ptr,$cnt,$rounds)=map("r$_",(3..8));
62my ($zero,$in0,$in1,$key,$rcon,$mask,$tmp)=map("v$_",(0..6));
63my ($stage,$outperm,$outmask,$outhead,$outtail)=map("v$_",(7..11));
64
65$code.=<<___;
66.machine "any"
67
68.text
69
70.align 7
71rcon:
72.long 0x01000000, 0x01000000, 0x01000000, 0x01000000 ?rev
73.long 0x1b000000, 0x1b000000, 0x1b000000, 0x1b000000 ?rev
74.long 0x0d0e0f0c, 0x0d0e0f0c, 0x0d0e0f0c, 0x0d0e0f0c ?rev
75.long 0,0,0,0 ?asis
76Lconsts:
77 mflr r0
78 bcl 20,31,\$+4
79 mflr $ptr #vvvvv "distance between . and rcon
80 addi $ptr,$ptr,-0x48
81 mtlr r0
82 blr
83 .long 0
84 .byte 0,12,0x14,0,0,0,0,0
85.asciz "AES for PowerISA 2.07, CRYPTOGAMS by <appro\@openssl.org>"
86
87.globl .${prefix}_set_encrypt_key
88Lset_encrypt_key:
89 mflr r11
90 $PUSH r11,$LRSAVE($sp)
91
92 li $ptr,-1
93 ${UCMP}i $inp,0
94 beq- Lenc_key_abort # if ($inp==0) return -1;
95 ${UCMP}i $out,0
96 beq- Lenc_key_abort # if ($out==0) return -1;
97 li $ptr,-2
98 cmpwi $bits,128
99 blt- Lenc_key_abort
100 cmpwi $bits,256
101 bgt- Lenc_key_abort
102 andi. r0,$bits,0x3f
103 bne- Lenc_key_abort
104
105 lis r0,0xfff0
106 mfspr $vrsave,256
107 mtspr 256,r0
108
109 bl Lconsts
110 mtlr r11
111
112 neg r9,$inp
113 lvx $in0,0,$inp
114 addi $inp,$inp,15 # 15 is not typo
115 lvsr $key,0,r9 # borrow $key
116 li r8,0x20
117 cmpwi $bits,192
118 lvx $in1,0,$inp
119 le?vspltisb $mask,0x0f # borrow $mask
120 lvx $rcon,0,$ptr
121 le?vxor $key,$key,$mask # adjust for byte swap
122 lvx $mask,r8,$ptr
123 addi $ptr,$ptr,0x10
124 vperm $in0,$in0,$in1,$key # align [and byte swap in LE]
125 li $cnt,8
126 vxor $zero,$zero,$zero
127 mtctr $cnt
128
129 ?lvsr $outperm,0,$out
130 vspltisb $outmask,-1
131 lvx $outhead,0,$out
132 ?vperm $outmask,$zero,$outmask,$outperm
133
134 blt Loop128
135 addi $inp,$inp,8
136 beq L192
137 addi $inp,$inp,8
138 b L256
139
140.align 4
141Loop128:
142 vperm $key,$in0,$in0,$mask # rotate-n-splat
143 vsldoi $tmp,$zero,$in0,12 # >>32
144 vperm $outtail,$in0,$in0,$outperm # rotate
145 vsel $stage,$outhead,$outtail,$outmask
146 vmr $outhead,$outtail
147 vcipherlast $key,$key,$rcon
148 stvx $stage,0,$out
149 addi $out,$out,16
150
151 vxor $in0,$in0,$tmp
152 vsldoi $tmp,$zero,$tmp,12 # >>32
153 vxor $in0,$in0,$tmp
154 vsldoi $tmp,$zero,$tmp,12 # >>32
155 vxor $in0,$in0,$tmp
156 vadduwm $rcon,$rcon,$rcon
157 vxor $in0,$in0,$key
158 bdnz Loop128
159
160 lvx $rcon,0,$ptr # last two round keys
161
162 vperm $key,$in0,$in0,$mask # rotate-n-splat
163 vsldoi $tmp,$zero,$in0,12 # >>32
164 vperm $outtail,$in0,$in0,$outperm # rotate
165 vsel $stage,$outhead,$outtail,$outmask
166 vmr $outhead,$outtail
167 vcipherlast $key,$key,$rcon
168 stvx $stage,0,$out
169 addi $out,$out,16
170
171 vxor $in0,$in0,$tmp
172 vsldoi $tmp,$zero,$tmp,12 # >>32
173 vxor $in0,$in0,$tmp
174 vsldoi $tmp,$zero,$tmp,12 # >>32
175 vxor $in0,$in0,$tmp
176 vadduwm $rcon,$rcon,$rcon
177 vxor $in0,$in0,$key
178
179 vperm $key,$in0,$in0,$mask # rotate-n-splat
180 vsldoi $tmp,$zero,$in0,12 # >>32
181 vperm $outtail,$in0,$in0,$outperm # rotate
182 vsel $stage,$outhead,$outtail,$outmask
183 vmr $outhead,$outtail
184 vcipherlast $key,$key,$rcon
185 stvx $stage,0,$out
186 addi $out,$out,16
187
188 vxor $in0,$in0,$tmp
189 vsldoi $tmp,$zero,$tmp,12 # >>32
190 vxor $in0,$in0,$tmp
191 vsldoi $tmp,$zero,$tmp,12 # >>32
192 vxor $in0,$in0,$tmp
193 vxor $in0,$in0,$key
194 vperm $outtail,$in0,$in0,$outperm # rotate
195 vsel $stage,$outhead,$outtail,$outmask
196 vmr $outhead,$outtail
197 stvx $stage,0,$out
198
199 addi $inp,$out,15 # 15 is not typo
200 addi $out,$out,0x50
201
202 li $rounds,10
203 b Ldone
204
205.align 4
206L192:
207 lvx $tmp,0,$inp
208 li $cnt,4
209 vperm $outtail,$in0,$in0,$outperm # rotate
210 vsel $stage,$outhead,$outtail,$outmask
211 vmr $outhead,$outtail
212 stvx $stage,0,$out
213 addi $out,$out,16
214 vperm $in1,$in1,$tmp,$key # align [and byte swap in LE]
215 vspltisb $key,8 # borrow $key
216 mtctr $cnt
217 vsububm $mask,$mask,$key # adjust the mask
218
219Loop192:
220 vperm $key,$in1,$in1,$mask # roate-n-splat
221 vsldoi $tmp,$zero,$in0,12 # >>32
222 vcipherlast $key,$key,$rcon
223
224 vxor $in0,$in0,$tmp
225 vsldoi $tmp,$zero,$tmp,12 # >>32
226 vxor $in0,$in0,$tmp
227 vsldoi $tmp,$zero,$tmp,12 # >>32
228 vxor $in0,$in0,$tmp
229
230 vsldoi $stage,$zero,$in1,8
231 vspltw $tmp,$in0,3
232 vxor $tmp,$tmp,$in1
233 vsldoi $in1,$zero,$in1,12 # >>32
234 vadduwm $rcon,$rcon,$rcon
235 vxor $in1,$in1,$tmp
236 vxor $in0,$in0,$key
237 vxor $in1,$in1,$key
238 vsldoi $stage,$stage,$in0,8
239
240 vperm $key,$in1,$in1,$mask # rotate-n-splat
241 vsldoi $tmp,$zero,$in0,12 # >>32
242 vperm $outtail,$stage,$stage,$outperm # rotate
243 vsel $stage,$outhead,$outtail,$outmask
244 vmr $outhead,$outtail
245 vcipherlast $key,$key,$rcon
246 stvx $stage,0,$out
247 addi $out,$out,16
248
249 vsldoi $stage,$in0,$in1,8
250 vxor $in0,$in0,$tmp
251 vsldoi $tmp,$zero,$tmp,12 # >>32
252 vperm $outtail,$stage,$stage,$outperm # rotate
253 vsel $stage,$outhead,$outtail,$outmask
254 vmr $outhead,$outtail
255 vxor $in0,$in0,$tmp
256 vsldoi $tmp,$zero,$tmp,12 # >>32
257 vxor $in0,$in0,$tmp
258 stvx $stage,0,$out
259 addi $out,$out,16
260
261 vspltw $tmp,$in0,3
262 vxor $tmp,$tmp,$in1
263 vsldoi $in1,$zero,$in1,12 # >>32
264 vadduwm $rcon,$rcon,$rcon
265 vxor $in1,$in1,$tmp
266 vxor $in0,$in0,$key
267 vxor $in1,$in1,$key
268 vperm $outtail,$in0,$in0,$outperm # rotate
269 vsel $stage,$outhead,$outtail,$outmask
270 vmr $outhead,$outtail
271 stvx $stage,0,$out
272 addi $inp,$out,15 # 15 is not typo
273 addi $out,$out,16
274 bdnz Loop192
275
276 li $rounds,12
277 addi $out,$out,0x20
278 b Ldone
279
280.align 4
281L256:
282 lvx $tmp,0,$inp
283 li $cnt,7
284 li $rounds,14
285 vperm $outtail,$in0,$in0,$outperm # rotate
286 vsel $stage,$outhead,$outtail,$outmask
287 vmr $outhead,$outtail
288 stvx $stage,0,$out
289 addi $out,$out,16
290 vperm $in1,$in1,$tmp,$key # align [and byte swap in LE]
291 mtctr $cnt
292
293Loop256:
294 vperm $key,$in1,$in1,$mask # rotate-n-splat
295 vsldoi $tmp,$zero,$in0,12 # >>32
296 vperm $outtail,$in1,$in1,$outperm # rotate
297 vsel $stage,$outhead,$outtail,$outmask
298 vmr $outhead,$outtail
299 vcipherlast $key,$key,$rcon
300 stvx $stage,0,$out
301 addi $out,$out,16
302
303 vxor $in0,$in0,$tmp
304 vsldoi $tmp,$zero,$tmp,12 # >>32
305 vxor $in0,$in0,$tmp
306 vsldoi $tmp,$zero,$tmp,12 # >>32
307 vxor $in0,$in0,$tmp
308 vadduwm $rcon,$rcon,$rcon
309 vxor $in0,$in0,$key
310 vperm $outtail,$in0,$in0,$outperm # rotate
311 vsel $stage,$outhead,$outtail,$outmask
312 vmr $outhead,$outtail
313 stvx $stage,0,$out
314 addi $inp,$out,15 # 15 is not typo
315 addi $out,$out,16
316 bdz Ldone
317
318 vspltw $key,$in0,3 # just splat
319 vsldoi $tmp,$zero,$in1,12 # >>32
320 vsbox $key,$key
321
322 vxor $in1,$in1,$tmp
323 vsldoi $tmp,$zero,$tmp,12 # >>32
324 vxor $in1,$in1,$tmp
325 vsldoi $tmp,$zero,$tmp,12 # >>32
326 vxor $in1,$in1,$tmp
327
328 vxor $in1,$in1,$key
329 b Loop256
330
331.align 4
332Ldone:
333 lvx $in1,0,$inp # redundant in aligned case
334 vsel $in1,$outhead,$in1,$outmask
335 stvx $in1,0,$inp
336 li $ptr,0
337 mtspr 256,$vrsave
338 stw $rounds,0($out)
339
340Lenc_key_abort:
341 mr r3,$ptr
342 blr
343 .long 0
344 .byte 0,12,0x14,1,0,0,3,0
345 .long 0
346.size .${prefix}_set_encrypt_key,.-.${prefix}_set_encrypt_key
347
348.globl .${prefix}_set_decrypt_key
349 $STU $sp,-$FRAME($sp)
350 mflr r10
351 $PUSH r10,$FRAME+$LRSAVE($sp)
352 bl Lset_encrypt_key
353 mtlr r10
354
355 cmpwi r3,0
356 bne- Ldec_key_abort
357
358 slwi $cnt,$rounds,4
359 subi $inp,$out,240 # first round key
360 srwi $rounds,$rounds,1
361 add $out,$inp,$cnt # last round key
362 mtctr $rounds
363
364Ldeckey:
365 lwz r0, 0($inp)
366 lwz r6, 4($inp)
367 lwz r7, 8($inp)
368 lwz r8, 12($inp)
369 addi $inp,$inp,16
370 lwz r9, 0($out)
371 lwz r10,4($out)
372 lwz r11,8($out)
373 lwz r12,12($out)
374 stw r0, 0($out)
375 stw r6, 4($out)
376 stw r7, 8($out)
377 stw r8, 12($out)
378 subi $out,$out,16
379 stw r9, -16($inp)
380 stw r10,-12($inp)
381 stw r11,-8($inp)
382 stw r12,-4($inp)
383 bdnz Ldeckey
384
385 xor r3,r3,r3 # return value
386Ldec_key_abort:
387 addi $sp,$sp,$FRAME
388 blr
389 .long 0
390 .byte 0,12,4,1,0x80,0,3,0
391 .long 0
392.size .${prefix}_set_decrypt_key,.-.${prefix}_set_decrypt_key
393___
394}}}
395#########################################################################
396{{{ # Single block en- and decrypt procedures #
397sub gen_block () {
398my $dir = shift;
399my $n = $dir eq "de" ? "n" : "";
400my ($inp,$out,$key,$rounds,$idx)=map("r$_",(3..7));
401
402$code.=<<___;
403.globl .${prefix}_${dir}crypt
404 lwz $rounds,240($key)
405 lis r0,0xfc00
406 mfspr $vrsave,256
407 li $idx,15 # 15 is not typo
408 mtspr 256,r0
409
410 lvx v0,0,$inp
411 neg r11,$out
412 lvx v1,$idx,$inp
413 lvsl v2,0,$inp # inpperm
414 le?vspltisb v4,0x0f
415 ?lvsl v3,0,r11 # outperm
416 le?vxor v2,v2,v4
417 li $idx,16
418 vperm v0,v0,v1,v2 # align [and byte swap in LE]
419 lvx v1,0,$key
420 ?lvsl v5,0,$key # keyperm
421 srwi $rounds,$rounds,1
422 lvx v2,$idx,$key
423 addi $idx,$idx,16
424 subi $rounds,$rounds,1
425 ?vperm v1,v1,v2,v5 # align round key
426
427 vxor v0,v0,v1
428 lvx v1,$idx,$key
429 addi $idx,$idx,16
430 mtctr $rounds
431
432Loop_${dir}c:
433 ?vperm v2,v2,v1,v5
434 v${n}cipher v0,v0,v2
435 lvx v2,$idx,$key
436 addi $idx,$idx,16
437 ?vperm v1,v1,v2,v5
438 v${n}cipher v0,v0,v1
439 lvx v1,$idx,$key
440 addi $idx,$idx,16
441 bdnz Loop_${dir}c
442
443 ?vperm v2,v2,v1,v5
444 v${n}cipher v0,v0,v2
445 lvx v2,$idx,$key
446 ?vperm v1,v1,v2,v5
447 v${n}cipherlast v0,v0,v1
448
449 vspltisb v2,-1
450 vxor v1,v1,v1
451 li $idx,15 # 15 is not typo
452 ?vperm v2,v1,v2,v3 # outmask
453 le?vxor v3,v3,v4
454 lvx v1,0,$out # outhead
455 vperm v0,v0,v0,v3 # rotate [and byte swap in LE]
456 vsel v1,v1,v0,v2
457 lvx v4,$idx,$out
458 stvx v1,0,$out
459 vsel v0,v0,v4,v2
460 stvx v0,$idx,$out
461
462 mtspr 256,$vrsave
463 blr
464 .long 0
465 .byte 0,12,0x14,0,0,0,3,0
466 .long 0
467.size .${prefix}_${dir}crypt,.-.${prefix}_${dir}crypt
468___
469}
470&gen_block("en");
471&gen_block("de");
472}}}
473#########################################################################
474{{{ # CBC en- and decrypt procedures #
475my ($inp,$out,$len,$key,$ivp,$enc,$rounds,$idx)=map("r$_",(3..10));
476my ($rndkey0,$rndkey1,$inout,$tmp)= map("v$_",(0..3));
477my ($ivec,$inptail,$inpperm,$outhead,$outperm,$outmask,$keyperm)=
478 map("v$_",(4..10));
479$code.=<<___;
480.globl .${prefix}_cbc_encrypt
481 ${UCMP}i $len,16
482 bltlr-
483
484 cmpwi $enc,0 # test direction
485 lis r0,0xffe0
486 mfspr $vrsave,256
487 mtspr 256,r0
488
489 li $idx,15
490 vxor $rndkey0,$rndkey0,$rndkey0
491 le?vspltisb $tmp,0x0f
492
493 lvx $ivec,0,$ivp # load [unaligned] iv
494 lvsl $inpperm,0,$ivp
495 lvx $inptail,$idx,$ivp
496 le?vxor $inpperm,$inpperm,$tmp
497 vperm $ivec,$ivec,$inptail,$inpperm
498
499 neg r11,$inp
500 ?lvsl $keyperm,0,$key # prepare for unaligned key
501 lwz $rounds,240($key)
502
503 lvsr $inpperm,0,r11 # prepare for unaligned load
504 lvx $inptail,0,$inp
505 addi $inp,$inp,15 # 15 is not typo
506 le?vxor $inpperm,$inpperm,$tmp
507
508 ?lvsr $outperm,0,$out # prepare for unaligned store
509 vspltisb $outmask,-1
510 lvx $outhead,0,$out
511 ?vperm $outmask,$rndkey0,$outmask,$outperm
512 le?vxor $outperm,$outperm,$tmp
513
514 srwi $rounds,$rounds,1
515 li $idx,16
516 subi $rounds,$rounds,1
517 beq Lcbc_dec
518
519Lcbc_enc:
520 vmr $inout,$inptail
521 lvx $inptail,0,$inp
522 addi $inp,$inp,16
523 mtctr $rounds
524 subi $len,$len,16 # len-=16
525
526 lvx $rndkey0,0,$key
527 vperm $inout,$inout,$inptail,$inpperm
528 lvx $rndkey1,$idx,$key
529 addi $idx,$idx,16
530 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
531 vxor $inout,$inout,$rndkey0
532 lvx $rndkey0,$idx,$key
533 addi $idx,$idx,16
534 vxor $inout,$inout,$ivec
535
536Loop_cbc_enc:
537 ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm
538 vcipher $inout,$inout,$rndkey1
539 lvx $rndkey1,$idx,$key
540 addi $idx,$idx,16
541 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
542 vcipher $inout,$inout,$rndkey0
543 lvx $rndkey0,$idx,$key
544 addi $idx,$idx,16
545 bdnz Loop_cbc_enc
546
547 ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm
548 vcipher $inout,$inout,$rndkey1
549 lvx $rndkey1,$idx,$key
550 li $idx,16
551 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
552 vcipherlast $ivec,$inout,$rndkey0
553 ${UCMP}i $len,16
554
555 vperm $tmp,$ivec,$ivec,$outperm
556 vsel $inout,$outhead,$tmp,$outmask
557 vmr $outhead,$tmp
558 stvx $inout,0,$out
559 addi $out,$out,16
560 bge Lcbc_enc
561
562 b Lcbc_done
563
564.align 4
565Lcbc_dec:
566 ${UCMP}i $len,128
567 bge _aesp8_cbc_decrypt8x
568 vmr $tmp,$inptail
569 lvx $inptail,0,$inp
570 addi $inp,$inp,16
571 mtctr $rounds
572 subi $len,$len,16 # len-=16
573
574 lvx $rndkey0,0,$key
575 vperm $tmp,$tmp,$inptail,$inpperm
576 lvx $rndkey1,$idx,$key
577 addi $idx,$idx,16
578 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
579 vxor $inout,$tmp,$rndkey0
580 lvx $rndkey0,$idx,$key
581 addi $idx,$idx,16
582
583Loop_cbc_dec:
584 ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm
585 vncipher $inout,$inout,$rndkey1
586 lvx $rndkey1,$idx,$key
587 addi $idx,$idx,16
588 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
589 vncipher $inout,$inout,$rndkey0
590 lvx $rndkey0,$idx,$key
591 addi $idx,$idx,16
592 bdnz Loop_cbc_dec
593
594 ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm
595 vncipher $inout,$inout,$rndkey1
596 lvx $rndkey1,$idx,$key
597 li $idx,16
598 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
599 vncipherlast $inout,$inout,$rndkey0
600 ${UCMP}i $len,16
601
602 vxor $inout,$inout,$ivec
603 vmr $ivec,$tmp
604 vperm $tmp,$inout,$inout,$outperm
605 vsel $inout,$outhead,$tmp,$outmask
606 vmr $outhead,$tmp
607 stvx $inout,0,$out
608 addi $out,$out,16
609 bge Lcbc_dec
610
611Lcbc_done:
612 addi $out,$out,-1
613 lvx $inout,0,$out # redundant in aligned case
614 vsel $inout,$outhead,$inout,$outmask
615 stvx $inout,0,$out
616
617 neg $enc,$ivp # write [unaligned] iv
618 li $idx,15 # 15 is not typo
619 vxor $rndkey0,$rndkey0,$rndkey0
620 vspltisb $outmask,-1
621 le?vspltisb $tmp,0x0f
622 ?lvsl $outperm,0,$enc
623 ?vperm $outmask,$rndkey0,$outmask,$outperm
624 le?vxor $outperm,$outperm,$tmp
625 lvx $outhead,0,$ivp
626 vperm $ivec,$ivec,$ivec,$outperm
627 vsel $inout,$outhead,$ivec,$outmask
628 lvx $inptail,$idx,$ivp
629 stvx $inout,0,$ivp
630 vsel $inout,$ivec,$inptail,$outmask
631 stvx $inout,$idx,$ivp
632
633 mtspr 256,$vrsave
634 blr
635 .long 0
636 .byte 0,12,0x14,0,0,0,6,0
637 .long 0
638___
639#########################################################################
640{{ # Optimized CBC decrypt procedure #
641my $key_="r11";
642my ($x00,$x10,$x20,$x30,$x40,$x50,$x60,$x70)=map("r$_",(0,8,26..31));
643my ($in0, $in1, $in2, $in3, $in4, $in5, $in6, $in7 )=map("v$_",(0..3,10..13));
644my ($out0,$out1,$out2,$out3,$out4,$out5,$out6,$out7)=map("v$_",(14..21));
645my $rndkey0="v23"; # v24-v25 rotating buffer for first found keys
646 # v26-v31 last 6 round keys
647my ($tmp,$keyperm)=($in3,$in4); # aliases with "caller", redundant assignment
648
649$code.=<<___;
650.align 5
651_aesp8_cbc_decrypt8x:
652 $STU $sp,-`($FRAME+21*16+6*$SIZE_T)`($sp)
653 li r10,`$FRAME+8*16+15`
654 li r11,`$FRAME+8*16+31`
655 stvx v20,r10,$sp # ABI says so
656 addi r10,r10,32
657 stvx v21,r11,$sp
658 addi r11,r11,32
659 stvx v22,r10,$sp
660 addi r10,r10,32
661 stvx v23,r11,$sp
662 addi r11,r11,32
663 stvx v24,r10,$sp
664 addi r10,r10,32
665 stvx v25,r11,$sp
666 addi r11,r11,32
667 stvx v26,r10,$sp
668 addi r10,r10,32
669 stvx v27,r11,$sp
670 addi r11,r11,32
671 stvx v28,r10,$sp
672 addi r10,r10,32
673 stvx v29,r11,$sp
674 addi r11,r11,32
675 stvx v30,r10,$sp
676 stvx v31,r11,$sp
677 li r0,-1
678 stw $vrsave,`$FRAME+21*16-4`($sp) # save vrsave
679 li $x10,0x10
680 $PUSH r26,`$FRAME+21*16+0*$SIZE_T`($sp)
681 li $x20,0x20
682 $PUSH r27,`$FRAME+21*16+1*$SIZE_T`($sp)
683 li $x30,0x30
684 $PUSH r28,`$FRAME+21*16+2*$SIZE_T`($sp)
685 li $x40,0x40
686 $PUSH r29,`$FRAME+21*16+3*$SIZE_T`($sp)
687 li $x50,0x50
688 $PUSH r30,`$FRAME+21*16+4*$SIZE_T`($sp)
689 li $x60,0x60
690 $PUSH r31,`$FRAME+21*16+5*$SIZE_T`($sp)
691 li $x70,0x70
692 mtspr 256,r0
693
694 subi $rounds,$rounds,3 # -4 in total
695 subi $len,$len,128 # bias
696
697 lvx $rndkey0,$x00,$key # load key schedule
698 lvx v30,$x10,$key
699 addi $key,$key,0x20
700 lvx v31,$x00,$key
701 ?vperm $rndkey0,$rndkey0,v30,$keyperm
702 addi $key_,$sp,$FRAME+15
703 mtctr $rounds
704
705Load_cbc_dec_key:
706 ?vperm v24,v30,v31,$keyperm
707 lvx v30,$x10,$key
708 addi $key,$key,0x20
709 stvx v24,$x00,$key_ # off-load round[1]
710 ?vperm v25,v31,v30,$keyperm
711 lvx v31,$x00,$key
712 stvx v25,$x10,$key_ # off-load round[2]
713 addi $key_,$key_,0x20
714 bdnz Load_cbc_dec_key
715
716 lvx v26,$x10,$key
717 ?vperm v24,v30,v31,$keyperm
718 lvx v27,$x20,$key
719 stvx v24,$x00,$key_ # off-load round[3]
720 ?vperm v25,v31,v26,$keyperm
721 lvx v28,$x30,$key
722 stvx v25,$x10,$key_ # off-load round[4]
723 addi $key_,$sp,$FRAME+15 # rewind $key_
724 ?vperm v26,v26,v27,$keyperm
725 lvx v29,$x40,$key
726 ?vperm v27,v27,v28,$keyperm
727 lvx v30,$x50,$key
728 ?vperm v28,v28,v29,$keyperm
729 lvx v31,$x60,$key
730 ?vperm v29,v29,v30,$keyperm
731 lvx $out0,$x70,$key # borrow $out0
732 ?vperm v30,v30,v31,$keyperm
733 lvx v24,$x00,$key_ # pre-load round[1]
734 ?vperm v31,v31,$out0,$keyperm
735 lvx v25,$x10,$key_ # pre-load round[2]
736
737 #lvx $inptail,0,$inp # "caller" already did this
738 #addi $inp,$inp,15 # 15 is not typo
739 subi $inp,$inp,15 # undo "caller"
740
741 le?li $idx,8
742 lvx_u $in0,$x00,$inp # load first 8 "words"
743 le?lvsl $inpperm,0,$idx
744 le?vspltisb $tmp,0x0f
745 lvx_u $in1,$x10,$inp
746 le?vxor $inpperm,$inpperm,$tmp # transform for lvx_u/stvx_u
747 lvx_u $in2,$x20,$inp
748 le?vperm $in0,$in0,$in0,$inpperm
749 lvx_u $in3,$x30,$inp
750 le?vperm $in1,$in1,$in1,$inpperm
751 lvx_u $in4,$x40,$inp
752 le?vperm $in2,$in2,$in2,$inpperm
753 vxor $out0,$in0,$rndkey0
754 lvx_u $in5,$x50,$inp
755 le?vperm $in3,$in3,$in3,$inpperm
756 vxor $out1,$in1,$rndkey0
757 lvx_u $in6,$x60,$inp
758 le?vperm $in4,$in4,$in4,$inpperm
759 vxor $out2,$in2,$rndkey0
760 lvx_u $in7,$x70,$inp
761 addi $inp,$inp,0x80
762 le?vperm $in5,$in5,$in5,$inpperm
763 vxor $out3,$in3,$rndkey0
764 le?vperm $in6,$in6,$in6,$inpperm
765 vxor $out4,$in4,$rndkey0
766 le?vperm $in7,$in7,$in7,$inpperm
767 vxor $out5,$in5,$rndkey0
768 vxor $out6,$in6,$rndkey0
769 vxor $out7,$in7,$rndkey0
770
771 mtctr $rounds
772 b Loop_cbc_dec8x
773.align 5
774Loop_cbc_dec8x:
775 vncipher $out0,$out0,v24
776 vncipher $out1,$out1,v24
777 vncipher $out2,$out2,v24
778 vncipher $out3,$out3,v24
779 vncipher $out4,$out4,v24
780 vncipher $out5,$out5,v24
781 vncipher $out6,$out6,v24
782 vncipher $out7,$out7,v24
783 lvx v24,$x20,$key_ # round[3]
784 addi $key_,$key_,0x20
785
786 vncipher $out0,$out0,v25
787 vncipher $out1,$out1,v25
788 vncipher $out2,$out2,v25
789 vncipher $out3,$out3,v25
790 vncipher $out4,$out4,v25
791 vncipher $out5,$out5,v25
792 vncipher $out6,$out6,v25
793 vncipher $out7,$out7,v25
794 lvx v25,$x10,$key_ # round[4]
795 bdnz Loop_cbc_dec8x
796
797 subic $len,$len,128 # $len-=128
798 vncipher $out0,$out0,v24
799 vncipher $out1,$out1,v24
800 vncipher $out2,$out2,v24
801 vncipher $out3,$out3,v24
802 vncipher $out4,$out4,v24
803 vncipher $out5,$out5,v24
804 vncipher $out6,$out6,v24
805 vncipher $out7,$out7,v24
806
807 subfe. r0,r0,r0 # borrow?-1:0
808 vncipher $out0,$out0,v25
809 vncipher $out1,$out1,v25
810 vncipher $out2,$out2,v25
811 vncipher $out3,$out3,v25
812 vncipher $out4,$out4,v25
813 vncipher $out5,$out5,v25
814 vncipher $out6,$out6,v25
815 vncipher $out7,$out7,v25
816
817 and r0,r0,$len
818 vncipher $out0,$out0,v26
819 vncipher $out1,$out1,v26
820 vncipher $out2,$out2,v26
821 vncipher $out3,$out3,v26
822 vncipher $out4,$out4,v26
823 vncipher $out5,$out5,v26
824 vncipher $out6,$out6,v26
825 vncipher $out7,$out7,v26
826
827 add $inp,$inp,r0 # $inp is adjusted in such
828 # way that at exit from the
829 # loop inX-in7 are loaded
830 # with last "words"
831 vncipher $out0,$out0,v27
832 vncipher $out1,$out1,v27
833 vncipher $out2,$out2,v27
834 vncipher $out3,$out3,v27
835 vncipher $out4,$out4,v27
836 vncipher $out5,$out5,v27
837 vncipher $out6,$out6,v27
838 vncipher $out7,$out7,v27
839
840 addi $key_,$sp,$FRAME+15 # rewind $key_
841 vncipher $out0,$out0,v28
842 vncipher $out1,$out1,v28
843 vncipher $out2,$out2,v28
844 vncipher $out3,$out3,v28
845 vncipher $out4,$out4,v28
846 vncipher $out5,$out5,v28
847 vncipher $out6,$out6,v28
848 vncipher $out7,$out7,v28
849 lvx v24,$x00,$key_ # re-pre-load round[1]
850
851 vncipher $out0,$out0,v29
852 vncipher $out1,$out1,v29
853 vncipher $out2,$out2,v29
854 vncipher $out3,$out3,v29
855 vncipher $out4,$out4,v29
856 vncipher $out5,$out5,v29
857 vncipher $out6,$out6,v29
858 vncipher $out7,$out7,v29
859 lvx v25,$x10,$key_ # re-pre-load round[2]
860
861 vncipher $out0,$out0,v30
862 vxor $ivec,$ivec,v31 # xor with last round key
863 vncipher $out1,$out1,v30
864 vxor $in0,$in0,v31
865 vncipher $out2,$out2,v30
866 vxor $in1,$in1,v31
867 vncipher $out3,$out3,v30
868 vxor $in2,$in2,v31
869 vncipher $out4,$out4,v30
870 vxor $in3,$in3,v31
871 vncipher $out5,$out5,v30
872 vxor $in4,$in4,v31
873 vncipher $out6,$out6,v30
874 vxor $in5,$in5,v31
875 vncipher $out7,$out7,v30
876 vxor $in6,$in6,v31
877
878 vncipherlast $out0,$out0,$ivec
879 vncipherlast $out1,$out1,$in0
880 lvx_u $in0,$x00,$inp # load next input block
881 vncipherlast $out2,$out2,$in1
882 lvx_u $in1,$x10,$inp
883 vncipherlast $out3,$out3,$in2
884 le?vperm $in0,$in0,$in0,$inpperm
885 lvx_u $in2,$x20,$inp
886 vncipherlast $out4,$out4,$in3
887 le?vperm $in1,$in1,$in1,$inpperm
888 lvx_u $in3,$x30,$inp
889 vncipherlast $out5,$out5,$in4
890 le?vperm $in2,$in2,$in2,$inpperm
891 lvx_u $in4,$x40,$inp
892 vncipherlast $out6,$out6,$in5
893 le?vperm $in3,$in3,$in3,$inpperm
894 lvx_u $in5,$x50,$inp
895 vncipherlast $out7,$out7,$in6
896 le?vperm $in4,$in4,$in4,$inpperm
897 lvx_u $in6,$x60,$inp
898 vmr $ivec,$in7
899 le?vperm $in5,$in5,$in5,$inpperm
900 lvx_u $in7,$x70,$inp
901 addi $inp,$inp,0x80
902
903 le?vperm $out0,$out0,$out0,$inpperm
904 le?vperm $out1,$out1,$out1,$inpperm
905 stvx_u $out0,$x00,$out
906 le?vperm $in6,$in6,$in6,$inpperm
907 vxor $out0,$in0,$rndkey0
908 le?vperm $out2,$out2,$out2,$inpperm
909 stvx_u $out1,$x10,$out
910 le?vperm $in7,$in7,$in7,$inpperm
911 vxor $out1,$in1,$rndkey0
912 le?vperm $out3,$out3,$out3,$inpperm
913 stvx_u $out2,$x20,$out
914 vxor $out2,$in2,$rndkey0
915 le?vperm $out4,$out4,$out4,$inpperm
916 stvx_u $out3,$x30,$out
917 vxor $out3,$in3,$rndkey0
918 le?vperm $out5,$out5,$out5,$inpperm
919 stvx_u $out4,$x40,$out
920 vxor $out4,$in4,$rndkey0
921 le?vperm $out6,$out6,$out6,$inpperm
922 stvx_u $out5,$x50,$out
923 vxor $out5,$in5,$rndkey0
924 le?vperm $out7,$out7,$out7,$inpperm
925 stvx_u $out6,$x60,$out
926 vxor $out6,$in6,$rndkey0
927 stvx_u $out7,$x70,$out
928 addi $out,$out,0x80
929 vxor $out7,$in7,$rndkey0
930
931 mtctr $rounds
932 beq Loop_cbc_dec8x # did $len-=128 borrow?
933
934 addic. $len,$len,128
935 beq Lcbc_dec8x_done
936 nop
937 nop
938
939Loop_cbc_dec8x_tail: # up to 7 "words" tail...
940 vncipher $out1,$out1,v24
941 vncipher $out2,$out2,v24
942 vncipher $out3,$out3,v24
943 vncipher $out4,$out4,v24
944 vncipher $out5,$out5,v24
945 vncipher $out6,$out6,v24
946 vncipher $out7,$out7,v24
947 lvx v24,$x20,$key_ # round[3]
948 addi $key_,$key_,0x20
949
950 vncipher $out1,$out1,v25
951 vncipher $out2,$out2,v25
952 vncipher $out3,$out3,v25
953 vncipher $out4,$out4,v25
954 vncipher $out5,$out5,v25
955 vncipher $out6,$out6,v25
956 vncipher $out7,$out7,v25
957 lvx v25,$x10,$key_ # round[4]
958 bdnz Loop_cbc_dec8x_tail
959
960 vncipher $out1,$out1,v24
961 vncipher $out2,$out2,v24
962 vncipher $out3,$out3,v24
963 vncipher $out4,$out4,v24
964 vncipher $out5,$out5,v24
965 vncipher $out6,$out6,v24
966 vncipher $out7,$out7,v24
967
968 vncipher $out1,$out1,v25
969 vncipher $out2,$out2,v25
970 vncipher $out3,$out3,v25
971 vncipher $out4,$out4,v25
972 vncipher $out5,$out5,v25
973 vncipher $out6,$out6,v25
974 vncipher $out7,$out7,v25
975
976 vncipher $out1,$out1,v26
977 vncipher $out2,$out2,v26
978 vncipher $out3,$out3,v26
979 vncipher $out4,$out4,v26
980 vncipher $out5,$out5,v26
981 vncipher $out6,$out6,v26
982 vncipher $out7,$out7,v26
983
984 vncipher $out1,$out1,v27
985 vncipher $out2,$out2,v27
986 vncipher $out3,$out3,v27
987 vncipher $out4,$out4,v27
988 vncipher $out5,$out5,v27
989 vncipher $out6,$out6,v27
990 vncipher $out7,$out7,v27
991
992 vncipher $out1,$out1,v28
993 vncipher $out2,$out2,v28
994 vncipher $out3,$out3,v28
995 vncipher $out4,$out4,v28
996 vncipher $out5,$out5,v28
997 vncipher $out6,$out6,v28
998 vncipher $out7,$out7,v28
999
1000 vncipher $out1,$out1,v29
1001 vncipher $out2,$out2,v29
1002 vncipher $out3,$out3,v29
1003 vncipher $out4,$out4,v29
1004 vncipher $out5,$out5,v29
1005 vncipher $out6,$out6,v29
1006 vncipher $out7,$out7,v29
1007
1008 vncipher $out1,$out1,v30
1009 vxor $ivec,$ivec,v31 # last round key
1010 vncipher $out2,$out2,v30
1011 vxor $in1,$in1,v31
1012 vncipher $out3,$out3,v30
1013 vxor $in2,$in2,v31
1014 vncipher $out4,$out4,v30
1015 vxor $in3,$in3,v31
1016 vncipher $out5,$out5,v30
1017 vxor $in4,$in4,v31
1018 vncipher $out6,$out6,v30
1019 vxor $in5,$in5,v31
1020 vncipher $out7,$out7,v30
1021 vxor $in6,$in6,v31
1022
1023 cmplwi $len,32 # switch($len)
1024 blt Lcbc_dec8x_one
1025 nop
1026 beq Lcbc_dec8x_two
1027 cmplwi $len,64
1028 blt Lcbc_dec8x_three
1029 nop
1030 beq Lcbc_dec8x_four
1031 cmplwi $len,96
1032 blt Lcbc_dec8x_five
1033 nop
1034 beq Lcbc_dec8x_six
1035
1036Lcbc_dec8x_seven:
1037 vncipherlast $out1,$out1,$ivec
1038 vncipherlast $out2,$out2,$in1
1039 vncipherlast $out3,$out3,$in2
1040 vncipherlast $out4,$out4,$in3
1041 vncipherlast $out5,$out5,$in4
1042 vncipherlast $out6,$out6,$in5
1043 vncipherlast $out7,$out7,$in6
1044 vmr $ivec,$in7
1045
1046 le?vperm $out1,$out1,$out1,$inpperm
1047 le?vperm $out2,$out2,$out2,$inpperm
1048 stvx_u $out1,$x00,$out
1049 le?vperm $out3,$out3,$out3,$inpperm
1050 stvx_u $out2,$x10,$out
1051 le?vperm $out4,$out4,$out4,$inpperm
1052 stvx_u $out3,$x20,$out
1053 le?vperm $out5,$out5,$out5,$inpperm
1054 stvx_u $out4,$x30,$out
1055 le?vperm $out6,$out6,$out6,$inpperm
1056 stvx_u $out5,$x40,$out
1057 le?vperm $out7,$out7,$out7,$inpperm
1058 stvx_u $out6,$x50,$out
1059 stvx_u $out7,$x60,$out
1060 addi $out,$out,0x70
1061 b Lcbc_dec8x_done
1062
1063.align 5
1064Lcbc_dec8x_six:
1065 vncipherlast $out2,$out2,$ivec
1066 vncipherlast $out3,$out3,$in2
1067 vncipherlast $out4,$out4,$in3
1068 vncipherlast $out5,$out5,$in4
1069 vncipherlast $out6,$out6,$in5
1070 vncipherlast $out7,$out7,$in6
1071 vmr $ivec,$in7
1072
1073 le?vperm $out2,$out2,$out2,$inpperm
1074 le?vperm $out3,$out3,$out3,$inpperm
1075 stvx_u $out2,$x00,$out
1076 le?vperm $out4,$out4,$out4,$inpperm
1077 stvx_u $out3,$x10,$out
1078 le?vperm $out5,$out5,$out5,$inpperm
1079 stvx_u $out4,$x20,$out
1080 le?vperm $out6,$out6,$out6,$inpperm
1081 stvx_u $out5,$x30,$out
1082 le?vperm $out7,$out7,$out7,$inpperm
1083 stvx_u $out6,$x40,$out
1084 stvx_u $out7,$x50,$out
1085 addi $out,$out,0x60
1086 b Lcbc_dec8x_done
1087
1088.align 5
1089Lcbc_dec8x_five:
1090 vncipherlast $out3,$out3,$ivec
1091 vncipherlast $out4,$out4,$in3
1092 vncipherlast $out5,$out5,$in4
1093 vncipherlast $out6,$out6,$in5
1094 vncipherlast $out7,$out7,$in6
1095 vmr $ivec,$in7
1096
1097 le?vperm $out3,$out3,$out3,$inpperm
1098 le?vperm $out4,$out4,$out4,$inpperm
1099 stvx_u $out3,$x00,$out
1100 le?vperm $out5,$out5,$out5,$inpperm
1101 stvx_u $out4,$x10,$out
1102 le?vperm $out6,$out6,$out6,$inpperm
1103 stvx_u $out5,$x20,$out
1104 le?vperm $out7,$out7,$out7,$inpperm
1105 stvx_u $out6,$x30,$out
1106 stvx_u $out7,$x40,$out
1107 addi $out,$out,0x50
1108 b Lcbc_dec8x_done
1109
1110.align 5
1111Lcbc_dec8x_four:
1112 vncipherlast $out4,$out4,$ivec
1113 vncipherlast $out5,$out5,$in4
1114 vncipherlast $out6,$out6,$in5
1115 vncipherlast $out7,$out7,$in6
1116 vmr $ivec,$in7
1117
1118 le?vperm $out4,$out4,$out4,$inpperm
1119 le?vperm $out5,$out5,$out5,$inpperm
1120 stvx_u $out4,$x00,$out
1121 le?vperm $out6,$out6,$out6,$inpperm
1122 stvx_u $out5,$x10,$out
1123 le?vperm $out7,$out7,$out7,$inpperm
1124 stvx_u $out6,$x20,$out
1125 stvx_u $out7,$x30,$out
1126 addi $out,$out,0x40
1127 b Lcbc_dec8x_done
1128
1129.align 5
1130Lcbc_dec8x_three:
1131 vncipherlast $out5,$out5,$ivec
1132 vncipherlast $out6,$out6,$in5
1133 vncipherlast $out7,$out7,$in6
1134 vmr $ivec,$in7
1135
1136 le?vperm $out5,$out5,$out5,$inpperm
1137 le?vperm $out6,$out6,$out6,$inpperm
1138 stvx_u $out5,$x00,$out
1139 le?vperm $out7,$out7,$out7,$inpperm
1140 stvx_u $out6,$x10,$out
1141 stvx_u $out7,$x20,$out
1142 addi $out,$out,0x30
1143 b Lcbc_dec8x_done
1144
1145.align 5
1146Lcbc_dec8x_two:
1147 vncipherlast $out6,$out6,$ivec
1148 vncipherlast $out7,$out7,$in6
1149 vmr $ivec,$in7
1150
1151 le?vperm $out6,$out6,$out6,$inpperm
1152 le?vperm $out7,$out7,$out7,$inpperm
1153 stvx_u $out6,$x00,$out
1154 stvx_u $out7,$x10,$out
1155 addi $out,$out,0x20
1156 b Lcbc_dec8x_done
1157
1158.align 5
1159Lcbc_dec8x_one:
1160 vncipherlast $out7,$out7,$ivec
1161 vmr $ivec,$in7
1162
1163 le?vperm $out7,$out7,$out7,$inpperm
1164 stvx_u $out7,0,$out
1165 addi $out,$out,0x10
1166
1167Lcbc_dec8x_done:
1168 le?vperm $ivec,$ivec,$ivec,$inpperm
1169 stvx_u $ivec,0,$ivp # write [unaligned] iv
1170
1171 li r10,`$FRAME+15`
1172 li r11,`$FRAME+31`
1173 stvx $inpperm,r10,$sp # wipe copies of round keys
1174 addi r10,r10,32
1175 stvx $inpperm,r11,$sp
1176 addi r11,r11,32
1177 stvx $inpperm,r10,$sp
1178 addi r10,r10,32
1179 stvx $inpperm,r11,$sp
1180 addi r11,r11,32
1181 stvx $inpperm,r10,$sp
1182 addi r10,r10,32
1183 stvx $inpperm,r11,$sp
1184 addi r11,r11,32
1185 stvx $inpperm,r10,$sp
1186 addi r10,r10,32
1187 stvx $inpperm,r11,$sp
1188 addi r11,r11,32
1189
1190 mtspr 256,$vrsave
1191 lvx v20,r10,$sp # ABI says so
1192 addi r10,r10,32
1193 lvx v21,r11,$sp
1194 addi r11,r11,32
1195 lvx v22,r10,$sp
1196 addi r10,r10,32
1197 lvx v23,r11,$sp
1198 addi r11,r11,32
1199 lvx v24,r10,$sp
1200 addi r10,r10,32
1201 lvx v25,r11,$sp
1202 addi r11,r11,32
1203 lvx v26,r10,$sp
1204 addi r10,r10,32
1205 lvx v27,r11,$sp
1206 addi r11,r11,32
1207 lvx v28,r10,$sp
1208 addi r10,r10,32
1209 lvx v29,r11,$sp
1210 addi r11,r11,32
1211 lvx v30,r10,$sp
1212 lvx v31,r11,$sp
1213 $POP r26,`$FRAME+21*16+0*$SIZE_T`($sp)
1214 $POP r27,`$FRAME+21*16+1*$SIZE_T`($sp)
1215 $POP r28,`$FRAME+21*16+2*$SIZE_T`($sp)
1216 $POP r29,`$FRAME+21*16+3*$SIZE_T`($sp)
1217 $POP r30,`$FRAME+21*16+4*$SIZE_T`($sp)
1218 $POP r31,`$FRAME+21*16+5*$SIZE_T`($sp)
1219 addi $sp,$sp,`$FRAME+21*16+6*$SIZE_T`
1220 blr
1221 .long 0
1222 .byte 0,12,0x14,0,0x80,6,6,0
1223 .long 0
1224.size .${prefix}_cbc_encrypt,.-.${prefix}_cbc_encrypt
1225___
1226}} }}}
1227
1228#########################################################################
1229{{{ # CTR procedure[s] #
1230my ($inp,$out,$len,$key,$ivp,$x10,$rounds,$idx)=map("r$_",(3..10));
1231my ($rndkey0,$rndkey1,$inout,$tmp)= map("v$_",(0..3));
1232my ($ivec,$inptail,$inpperm,$outhead,$outperm,$outmask,$keyperm,$one)=
1233 map("v$_",(4..11));
1234my $dat=$tmp;
1235
1236$code.=<<___;
1237.globl .${prefix}_ctr32_encrypt_blocks
1238 ${UCMP}i $len,1
1239 bltlr-
1240
1241 lis r0,0xfff0
1242 mfspr $vrsave,256
1243 mtspr 256,r0
1244
1245 li $idx,15
1246 vxor $rndkey0,$rndkey0,$rndkey0
1247 le?vspltisb $tmp,0x0f
1248
1249 lvx $ivec,0,$ivp # load [unaligned] iv
1250 lvsl $inpperm,0,$ivp
1251 lvx $inptail,$idx,$ivp
1252 vspltisb $one,1
1253 le?vxor $inpperm,$inpperm,$tmp
1254 vperm $ivec,$ivec,$inptail,$inpperm
1255 vsldoi $one,$rndkey0,$one,1
1256
1257 neg r11,$inp
1258 ?lvsl $keyperm,0,$key # prepare for unaligned key
1259 lwz $rounds,240($key)
1260
1261 lvsr $inpperm,0,r11 # prepare for unaligned load
1262 lvx $inptail,0,$inp
1263 addi $inp,$inp,15 # 15 is not typo
1264 le?vxor $inpperm,$inpperm,$tmp
1265
1266 srwi $rounds,$rounds,1
1267 li $idx,16
1268 subi $rounds,$rounds,1
1269
1270 ${UCMP}i $len,8
1271 bge _aesp8_ctr32_encrypt8x
1272
1273 ?lvsr $outperm,0,$out # prepare for unaligned store
1274 vspltisb $outmask,-1
1275 lvx $outhead,0,$out
1276 ?vperm $outmask,$rndkey0,$outmask,$outperm
1277 le?vxor $outperm,$outperm,$tmp
1278
1279 lvx $rndkey0,0,$key
1280 mtctr $rounds
1281 lvx $rndkey1,$idx,$key
1282 addi $idx,$idx,16
1283 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
1284 vxor $inout,$ivec,$rndkey0
1285 lvx $rndkey0,$idx,$key
1286 addi $idx,$idx,16
1287 b Loop_ctr32_enc
1288
1289.align 5
1290Loop_ctr32_enc:
1291 ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm
1292 vcipher $inout,$inout,$rndkey1
1293 lvx $rndkey1,$idx,$key
1294 addi $idx,$idx,16
1295 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
1296 vcipher $inout,$inout,$rndkey0
1297 lvx $rndkey0,$idx,$key
1298 addi $idx,$idx,16
1299 bdnz Loop_ctr32_enc
1300
1301 vadduwm $ivec,$ivec,$one
1302 vmr $dat,$inptail
1303 lvx $inptail,0,$inp
1304 addi $inp,$inp,16
1305 subic. $len,$len,1 # blocks--
1306
1307 ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm
1308 vcipher $inout,$inout,$rndkey1
1309 lvx $rndkey1,$idx,$key
1310 vperm $dat,$dat,$inptail,$inpperm
1311 li $idx,16
1312 ?vperm $rndkey1,$rndkey0,$rndkey1,$keyperm
1313 lvx $rndkey0,0,$key
1314 vxor $dat,$dat,$rndkey1 # last round key
1315 vcipherlast $inout,$inout,$dat
1316
1317 lvx $rndkey1,$idx,$key
1318 addi $idx,$idx,16
1319 vperm $inout,$inout,$inout,$outperm
1320 vsel $dat,$outhead,$inout,$outmask
1321 mtctr $rounds
1322 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
1323 vmr $outhead,$inout
1324 vxor $inout,$ivec,$rndkey0
1325 lvx $rndkey0,$idx,$key
1326 addi $idx,$idx,16
1327 stvx $dat,0,$out
1328 addi $out,$out,16
1329 bne Loop_ctr32_enc
1330
1331 addi $out,$out,-1
1332 lvx $inout,0,$out # redundant in aligned case
1333 vsel $inout,$outhead,$inout,$outmask
1334 stvx $inout,0,$out
1335
1336 mtspr 256,$vrsave
1337 blr
1338 .long 0
1339 .byte 0,12,0x14,0,0,0,6,0
1340 .long 0
1341___
1342#########################################################################
1343{{ # Optimized CTR procedure #
1344my $key_="r11";
1345my ($x00,$x10,$x20,$x30,$x40,$x50,$x60,$x70)=map("r$_",(0,8,26..31));
1346my ($in0, $in1, $in2, $in3, $in4, $in5, $in6, $in7 )=map("v$_",(0..3,10,12..14));
1347my ($out0,$out1,$out2,$out3,$out4,$out5,$out6,$out7)=map("v$_",(15..22));
1348my $rndkey0="v23"; # v24-v25 rotating buffer for first found keys
1349 # v26-v31 last 6 round keys
1350my ($tmp,$keyperm)=($in3,$in4); # aliases with "caller", redundant assignment
1351my ($two,$three,$four)=($outhead,$outperm,$outmask);
1352
1353$code.=<<___;
1354.align 5
1355_aesp8_ctr32_encrypt8x:
1356 $STU $sp,-`($FRAME+21*16+6*$SIZE_T)`($sp)
1357 li r10,`$FRAME+8*16+15`
1358 li r11,`$FRAME+8*16+31`
1359 stvx v20,r10,$sp # ABI says so
1360 addi r10,r10,32
1361 stvx v21,r11,$sp
1362 addi r11,r11,32
1363 stvx v22,r10,$sp
1364 addi r10,r10,32
1365 stvx v23,r11,$sp
1366 addi r11,r11,32
1367 stvx v24,r10,$sp
1368 addi r10,r10,32
1369 stvx v25,r11,$sp
1370 addi r11,r11,32
1371 stvx v26,r10,$sp
1372 addi r10,r10,32
1373 stvx v27,r11,$sp
1374 addi r11,r11,32
1375 stvx v28,r10,$sp
1376 addi r10,r10,32
1377 stvx v29,r11,$sp
1378 addi r11,r11,32
1379 stvx v30,r10,$sp
1380 stvx v31,r11,$sp
1381 li r0,-1
1382 stw $vrsave,`$FRAME+21*16-4`($sp) # save vrsave
1383 li $x10,0x10
1384 $PUSH r26,`$FRAME+21*16+0*$SIZE_T`($sp)
1385 li $x20,0x20
1386 $PUSH r27,`$FRAME+21*16+1*$SIZE_T`($sp)
1387 li $x30,0x30
1388 $PUSH r28,`$FRAME+21*16+2*$SIZE_T`($sp)
1389 li $x40,0x40
1390 $PUSH r29,`$FRAME+21*16+3*$SIZE_T`($sp)
1391 li $x50,0x50
1392 $PUSH r30,`$FRAME+21*16+4*$SIZE_T`($sp)
1393 li $x60,0x60
1394 $PUSH r31,`$FRAME+21*16+5*$SIZE_T`($sp)
1395 li $x70,0x70
1396 mtspr 256,r0
1397
1398 subi $rounds,$rounds,3 # -4 in total
1399
1400 lvx $rndkey0,$x00,$key # load key schedule
1401 lvx v30,$x10,$key
1402 addi $key,$key,0x20
1403 lvx v31,$x00,$key
1404 ?vperm $rndkey0,$rndkey0,v30,$keyperm
1405 addi $key_,$sp,$FRAME+15
1406 mtctr $rounds
1407
1408Load_ctr32_enc_key:
1409 ?vperm v24,v30,v31,$keyperm
1410 lvx v30,$x10,$key
1411 addi $key,$key,0x20
1412 stvx v24,$x00,$key_ # off-load round[1]
1413 ?vperm v25,v31,v30,$keyperm
1414 lvx v31,$x00,$key
1415 stvx v25,$x10,$key_ # off-load round[2]
1416 addi $key_,$key_,0x20
1417 bdnz Load_ctr32_enc_key
1418
1419 lvx v26,$x10,$key
1420 ?vperm v24,v30,v31,$keyperm
1421 lvx v27,$x20,$key
1422 stvx v24,$x00,$key_ # off-load round[3]
1423 ?vperm v25,v31,v26,$keyperm
1424 lvx v28,$x30,$key
1425 stvx v25,$x10,$key_ # off-load round[4]
1426 addi $key_,$sp,$FRAME+15 # rewind $key_
1427 ?vperm v26,v26,v27,$keyperm
1428 lvx v29,$x40,$key
1429 ?vperm v27,v27,v28,$keyperm
1430 lvx v30,$x50,$key
1431 ?vperm v28,v28,v29,$keyperm
1432 lvx v31,$x60,$key
1433 ?vperm v29,v29,v30,$keyperm
1434 lvx $out0,$x70,$key # borrow $out0
1435 ?vperm v30,v30,v31,$keyperm
1436 lvx v24,$x00,$key_ # pre-load round[1]
1437 ?vperm v31,v31,$out0,$keyperm
1438 lvx v25,$x10,$key_ # pre-load round[2]
1439
1440 vadduwm $two,$one,$one
1441 subi $inp,$inp,15 # undo "caller"
1442 $SHL $len,$len,4
1443
1444 vadduwm $out1,$ivec,$one # counter values ...
1445 vadduwm $out2,$ivec,$two
1446 vxor $out0,$ivec,$rndkey0 # ... xored with rndkey[0]
1447 le?li $idx,8
1448 vadduwm $out3,$out1,$two
1449 vxor $out1,$out1,$rndkey0
1450 le?lvsl $inpperm,0,$idx
1451 vadduwm $out4,$out2,$two
1452 vxor $out2,$out2,$rndkey0
1453 le?vspltisb $tmp,0x0f
1454 vadduwm $out5,$out3,$two
1455 vxor $out3,$out3,$rndkey0
1456 le?vxor $inpperm,$inpperm,$tmp # transform for lvx_u/stvx_u
1457 vadduwm $out6,$out4,$two
1458 vxor $out4,$out4,$rndkey0
1459 vadduwm $out7,$out5,$two
1460 vxor $out5,$out5,$rndkey0
1461 vadduwm $ivec,$out6,$two # next counter value
1462 vxor $out6,$out6,$rndkey0
1463 vxor $out7,$out7,$rndkey0
1464
1465 mtctr $rounds
1466 b Loop_ctr32_enc8x
1467.align 5
1468Loop_ctr32_enc8x:
1469 vcipher $out0,$out0,v24
1470 vcipher $out1,$out1,v24
1471 vcipher $out2,$out2,v24
1472 vcipher $out3,$out3,v24
1473 vcipher $out4,$out4,v24
1474 vcipher $out5,$out5,v24
1475 vcipher $out6,$out6,v24
1476 vcipher $out7,$out7,v24
1477Loop_ctr32_enc8x_middle:
1478 lvx v24,$x20,$key_ # round[3]
1479 addi $key_,$key_,0x20
1480
1481 vcipher $out0,$out0,v25
1482 vcipher $out1,$out1,v25
1483 vcipher $out2,$out2,v25
1484 vcipher $out3,$out3,v25
1485 vcipher $out4,$out4,v25
1486 vcipher $out5,$out5,v25
1487 vcipher $out6,$out6,v25
1488 vcipher $out7,$out7,v25
1489 lvx v25,$x10,$key_ # round[4]
1490 bdnz Loop_ctr32_enc8x
1491
1492 subic r11,$len,256 # $len-256, borrow $key_
1493 vcipher $out0,$out0,v24
1494 vcipher $out1,$out1,v24
1495 vcipher $out2,$out2,v24
1496 vcipher $out3,$out3,v24
1497 vcipher $out4,$out4,v24
1498 vcipher $out5,$out5,v24
1499 vcipher $out6,$out6,v24
1500 vcipher $out7,$out7,v24
1501
1502 subfe r0,r0,r0 # borrow?-1:0
1503 vcipher $out0,$out0,v25
1504 vcipher $out1,$out1,v25
1505 vcipher $out2,$out2,v25
1506 vcipher $out3,$out3,v25
1507 vcipher $out4,$out4,v25
1508 vcipher $out5,$out5,v25
1509 vcipher $out6,$out6,v25
1510 vcipher $out7,$out7,v25
1511
1512 and r0,r0,r11
1513 addi $key_,$sp,$FRAME+15 # rewind $key_
1514 vcipher $out0,$out0,v26
1515 vcipher $out1,$out1,v26
1516 vcipher $out2,$out2,v26
1517 vcipher $out3,$out3,v26
1518 vcipher $out4,$out4,v26
1519 vcipher $out5,$out5,v26
1520 vcipher $out6,$out6,v26
1521 vcipher $out7,$out7,v26
1522 lvx v24,$x00,$key_ # re-pre-load round[1]
1523
1524 subic $len,$len,129 # $len-=129
1525 vcipher $out0,$out0,v27
1526 addi $len,$len,1 # $len-=128 really
1527 vcipher $out1,$out1,v27
1528 vcipher $out2,$out2,v27
1529 vcipher $out3,$out3,v27
1530 vcipher $out4,$out4,v27
1531 vcipher $out5,$out5,v27
1532 vcipher $out6,$out6,v27
1533 vcipher $out7,$out7,v27
1534 lvx v25,$x10,$key_ # re-pre-load round[2]
1535
1536 vcipher $out0,$out0,v28
1537 lvx_u $in0,$x00,$inp # load input
1538 vcipher $out1,$out1,v28
1539 lvx_u $in1,$x10,$inp
1540 vcipher $out2,$out2,v28
1541 lvx_u $in2,$x20,$inp
1542 vcipher $out3,$out3,v28
1543 lvx_u $in3,$x30,$inp
1544 vcipher $out4,$out4,v28
1545 lvx_u $in4,$x40,$inp
1546 vcipher $out5,$out5,v28
1547 lvx_u $in5,$x50,$inp
1548 vcipher $out6,$out6,v28
1549 lvx_u $in6,$x60,$inp
1550 vcipher $out7,$out7,v28
1551 lvx_u $in7,$x70,$inp
1552 addi $inp,$inp,0x80
1553
1554 vcipher $out0,$out0,v29
1555 le?vperm $in0,$in0,$in0,$inpperm
1556 vcipher $out1,$out1,v29
1557 le?vperm $in1,$in1,$in1,$inpperm
1558 vcipher $out2,$out2,v29
1559 le?vperm $in2,$in2,$in2,$inpperm
1560 vcipher $out3,$out3,v29
1561 le?vperm $in3,$in3,$in3,$inpperm
1562 vcipher $out4,$out4,v29
1563 le?vperm $in4,$in4,$in4,$inpperm
1564 vcipher $out5,$out5,v29
1565 le?vperm $in5,$in5,$in5,$inpperm
1566 vcipher $out6,$out6,v29
1567 le?vperm $in6,$in6,$in6,$inpperm
1568 vcipher $out7,$out7,v29
1569 le?vperm $in7,$in7,$in7,$inpperm
1570
1571 add $inp,$inp,r0 # $inp is adjusted in such
1572 # way that at exit from the
1573 # loop inX-in7 are loaded
1574 # with last "words"
1575 subfe. r0,r0,r0 # borrow?-1:0
1576 vcipher $out0,$out0,v30
1577 vxor $in0,$in0,v31 # xor with last round key
1578 vcipher $out1,$out1,v30
1579 vxor $in1,$in1,v31
1580 vcipher $out2,$out2,v30
1581 vxor $in2,$in2,v31
1582 vcipher $out3,$out3,v30
1583 vxor $in3,$in3,v31
1584 vcipher $out4,$out4,v30
1585 vxor $in4,$in4,v31
1586 vcipher $out5,$out5,v30
1587 vxor $in5,$in5,v31
1588 vcipher $out6,$out6,v30
1589 vxor $in6,$in6,v31
1590 vcipher $out7,$out7,v30
1591 vxor $in7,$in7,v31
1592
1593 bne Lctr32_enc8x_break # did $len-129 borrow?
1594
1595 vcipherlast $in0,$out0,$in0
1596 vcipherlast $in1,$out1,$in1
1597 vadduwm $out1,$ivec,$one # counter values ...
1598 vcipherlast $in2,$out2,$in2
1599 vadduwm $out2,$ivec,$two
1600 vxor $out0,$ivec,$rndkey0 # ... xored with rndkey[0]
1601 vcipherlast $in3,$out3,$in3
1602 vadduwm $out3,$out1,$two
1603 vxor $out1,$out1,$rndkey0
1604 vcipherlast $in4,$out4,$in4
1605 vadduwm $out4,$out2,$two
1606 vxor $out2,$out2,$rndkey0
1607 vcipherlast $in5,$out5,$in5
1608 vadduwm $out5,$out3,$two
1609 vxor $out3,$out3,$rndkey0
1610 vcipherlast $in6,$out6,$in6
1611 vadduwm $out6,$out4,$two
1612 vxor $out4,$out4,$rndkey0
1613 vcipherlast $in7,$out7,$in7
1614 vadduwm $out7,$out5,$two
1615 vxor $out5,$out5,$rndkey0
1616 le?vperm $in0,$in0,$in0,$inpperm
1617 vadduwm $ivec,$out6,$two # next counter value
1618 vxor $out6,$out6,$rndkey0
1619 le?vperm $in1,$in1,$in1,$inpperm
1620 vxor $out7,$out7,$rndkey0
1621 mtctr $rounds
1622
1623 vcipher $out0,$out0,v24
1624 stvx_u $in0,$x00,$out
1625 le?vperm $in2,$in2,$in2,$inpperm
1626 vcipher $out1,$out1,v24
1627 stvx_u $in1,$x10,$out
1628 le?vperm $in3,$in3,$in3,$inpperm
1629 vcipher $out2,$out2,v24
1630 stvx_u $in2,$x20,$out
1631 le?vperm $in4,$in4,$in4,$inpperm
1632 vcipher $out3,$out3,v24
1633 stvx_u $in3,$x30,$out
1634 le?vperm $in5,$in5,$in5,$inpperm
1635 vcipher $out4,$out4,v24
1636 stvx_u $in4,$x40,$out
1637 le?vperm $in6,$in6,$in6,$inpperm
1638 vcipher $out5,$out5,v24
1639 stvx_u $in5,$x50,$out
1640 le?vperm $in7,$in7,$in7,$inpperm
1641 vcipher $out6,$out6,v24
1642 stvx_u $in6,$x60,$out
1643 vcipher $out7,$out7,v24
1644 stvx_u $in7,$x70,$out
1645 addi $out,$out,0x80
1646
1647 b Loop_ctr32_enc8x_middle
1648
1649.align 5
1650Lctr32_enc8x_break:
1651 cmpwi $len,-0x60
1652 blt Lctr32_enc8x_one
1653 nop
1654 beq Lctr32_enc8x_two
1655 cmpwi $len,-0x40
1656 blt Lctr32_enc8x_three
1657 nop
1658 beq Lctr32_enc8x_four
1659 cmpwi $len,-0x20
1660 blt Lctr32_enc8x_five
1661 nop
1662 beq Lctr32_enc8x_six
1663 cmpwi $len,0x00
1664 blt Lctr32_enc8x_seven
1665
1666Lctr32_enc8x_eight:
1667 vcipherlast $out0,$out0,$in0
1668 vcipherlast $out1,$out1,$in1
1669 vcipherlast $out2,$out2,$in2
1670 vcipherlast $out3,$out3,$in3
1671 vcipherlast $out4,$out4,$in4
1672 vcipherlast $out5,$out5,$in5
1673 vcipherlast $out6,$out6,$in6
1674 vcipherlast $out7,$out7,$in7
1675
1676 le?vperm $out0,$out0,$out0,$inpperm
1677 le?vperm $out1,$out1,$out1,$inpperm
1678 stvx_u $out0,$x00,$out
1679 le?vperm $out2,$out2,$out2,$inpperm
1680 stvx_u $out1,$x10,$out
1681 le?vperm $out3,$out3,$out3,$inpperm
1682 stvx_u $out2,$x20,$out
1683 le?vperm $out4,$out4,$out4,$inpperm
1684 stvx_u $out3,$x30,$out
1685 le?vperm $out5,$out5,$out5,$inpperm
1686 stvx_u $out4,$x40,$out
1687 le?vperm $out6,$out6,$out6,$inpperm
1688 stvx_u $out5,$x50,$out
1689 le?vperm $out7,$out7,$out7,$inpperm
1690 stvx_u $out6,$x60,$out
1691 stvx_u $out7,$x70,$out
1692 addi $out,$out,0x80
1693 b Lctr32_enc8x_done
1694
1695.align 5
1696Lctr32_enc8x_seven:
1697 vcipherlast $out0,$out0,$in1
1698 vcipherlast $out1,$out1,$in2
1699 vcipherlast $out2,$out2,$in3
1700 vcipherlast $out3,$out3,$in4
1701 vcipherlast $out4,$out4,$in5
1702 vcipherlast $out5,$out5,$in6
1703 vcipherlast $out6,$out6,$in7
1704
1705 le?vperm $out0,$out0,$out0,$inpperm
1706 le?vperm $out1,$out1,$out1,$inpperm
1707 stvx_u $out0,$x00,$out
1708 le?vperm $out2,$out2,$out2,$inpperm
1709 stvx_u $out1,$x10,$out
1710 le?vperm $out3,$out3,$out3,$inpperm
1711 stvx_u $out2,$x20,$out
1712 le?vperm $out4,$out4,$out4,$inpperm
1713 stvx_u $out3,$x30,$out
1714 le?vperm $out5,$out5,$out5,$inpperm
1715 stvx_u $out4,$x40,$out
1716 le?vperm $out6,$out6,$out6,$inpperm
1717 stvx_u $out5,$x50,$out
1718 stvx_u $out6,$x60,$out
1719 addi $out,$out,0x70
1720 b Lctr32_enc8x_done
1721
1722.align 5
1723Lctr32_enc8x_six:
1724 vcipherlast $out0,$out0,$in2
1725 vcipherlast $out1,$out1,$in3
1726 vcipherlast $out2,$out2,$in4
1727 vcipherlast $out3,$out3,$in5
1728 vcipherlast $out4,$out4,$in6
1729 vcipherlast $out5,$out5,$in7
1730
1731 le?vperm $out0,$out0,$out0,$inpperm
1732 le?vperm $out1,$out1,$out1,$inpperm
1733 stvx_u $out0,$x00,$out
1734 le?vperm $out2,$out2,$out2,$inpperm
1735 stvx_u $out1,$x10,$out
1736 le?vperm $out3,$out3,$out3,$inpperm
1737 stvx_u $out2,$x20,$out
1738 le?vperm $out4,$out4,$out4,$inpperm
1739 stvx_u $out3,$x30,$out
1740 le?vperm $out5,$out5,$out5,$inpperm
1741 stvx_u $out4,$x40,$out
1742 stvx_u $out5,$x50,$out
1743 addi $out,$out,0x60
1744 b Lctr32_enc8x_done
1745
1746.align 5
1747Lctr32_enc8x_five:
1748 vcipherlast $out0,$out0,$in3
1749 vcipherlast $out1,$out1,$in4
1750 vcipherlast $out2,$out2,$in5
1751 vcipherlast $out3,$out3,$in6
1752 vcipherlast $out4,$out4,$in7
1753
1754 le?vperm $out0,$out0,$out0,$inpperm
1755 le?vperm $out1,$out1,$out1,$inpperm
1756 stvx_u $out0,$x00,$out
1757 le?vperm $out2,$out2,$out2,$inpperm
1758 stvx_u $out1,$x10,$out
1759 le?vperm $out3,$out3,$out3,$inpperm
1760 stvx_u $out2,$x20,$out
1761 le?vperm $out4,$out4,$out4,$inpperm
1762 stvx_u $out3,$x30,$out
1763 stvx_u $out4,$x40,$out
1764 addi $out,$out,0x50
1765 b Lctr32_enc8x_done
1766
1767.align 5
1768Lctr32_enc8x_four:
1769 vcipherlast $out0,$out0,$in4
1770 vcipherlast $out1,$out1,$in5
1771 vcipherlast $out2,$out2,$in6
1772 vcipherlast $out3,$out3,$in7
1773
1774 le?vperm $out0,$out0,$out0,$inpperm
1775 le?vperm $out1,$out1,$out1,$inpperm
1776 stvx_u $out0,$x00,$out
1777 le?vperm $out2,$out2,$out2,$inpperm
1778 stvx_u $out1,$x10,$out
1779 le?vperm $out3,$out3,$out3,$inpperm
1780 stvx_u $out2,$x20,$out
1781 stvx_u $out3,$x30,$out
1782 addi $out,$out,0x40
1783 b Lctr32_enc8x_done
1784
1785.align 5
1786Lctr32_enc8x_three:
1787 vcipherlast $out0,$out0,$in5
1788 vcipherlast $out1,$out1,$in6
1789 vcipherlast $out2,$out2,$in7
1790
1791 le?vperm $out0,$out0,$out0,$inpperm
1792 le?vperm $out1,$out1,$out1,$inpperm
1793 stvx_u $out0,$x00,$out
1794 le?vperm $out2,$out2,$out2,$inpperm
1795 stvx_u $out1,$x10,$out
1796 stvx_u $out2,$x20,$out
1797 addi $out,$out,0x30
1798 b Lcbc_dec8x_done
1799
1800.align 5
1801Lctr32_enc8x_two:
1802 vcipherlast $out0,$out0,$in6
1803 vcipherlast $out1,$out1,$in7
1804
1805 le?vperm $out0,$out0,$out0,$inpperm
1806 le?vperm $out1,$out1,$out1,$inpperm
1807 stvx_u $out0,$x00,$out
1808 stvx_u $out1,$x10,$out
1809 addi $out,$out,0x20
1810 b Lcbc_dec8x_done
1811
1812.align 5
1813Lctr32_enc8x_one:
1814 vcipherlast $out0,$out0,$in7
1815
1816 le?vperm $out0,$out0,$out0,$inpperm
1817 stvx_u $out0,0,$out
1818 addi $out,$out,0x10
1819
1820Lctr32_enc8x_done:
1821 li r10,`$FRAME+15`
1822 li r11,`$FRAME+31`
1823 stvx $inpperm,r10,$sp # wipe copies of round keys
1824 addi r10,r10,32
1825 stvx $inpperm,r11,$sp
1826 addi r11,r11,32
1827 stvx $inpperm,r10,$sp
1828 addi r10,r10,32
1829 stvx $inpperm,r11,$sp
1830 addi r11,r11,32
1831 stvx $inpperm,r10,$sp
1832 addi r10,r10,32
1833 stvx $inpperm,r11,$sp
1834 addi r11,r11,32
1835 stvx $inpperm,r10,$sp
1836 addi r10,r10,32
1837 stvx $inpperm,r11,$sp
1838 addi r11,r11,32
1839
1840 mtspr 256,$vrsave
1841 lvx v20,r10,$sp # ABI says so
1842 addi r10,r10,32
1843 lvx v21,r11,$sp
1844 addi r11,r11,32
1845 lvx v22,r10,$sp
1846 addi r10,r10,32
1847 lvx v23,r11,$sp
1848 addi r11,r11,32
1849 lvx v24,r10,$sp
1850 addi r10,r10,32
1851 lvx v25,r11,$sp
1852 addi r11,r11,32
1853 lvx v26,r10,$sp
1854 addi r10,r10,32
1855 lvx v27,r11,$sp
1856 addi r11,r11,32
1857 lvx v28,r10,$sp
1858 addi r10,r10,32
1859 lvx v29,r11,$sp
1860 addi r11,r11,32
1861 lvx v30,r10,$sp
1862 lvx v31,r11,$sp
1863 $POP r26,`$FRAME+21*16+0*$SIZE_T`($sp)
1864 $POP r27,`$FRAME+21*16+1*$SIZE_T`($sp)
1865 $POP r28,`$FRAME+21*16+2*$SIZE_T`($sp)
1866 $POP r29,`$FRAME+21*16+3*$SIZE_T`($sp)
1867 $POP r30,`$FRAME+21*16+4*$SIZE_T`($sp)
1868 $POP r31,`$FRAME+21*16+5*$SIZE_T`($sp)
1869 addi $sp,$sp,`$FRAME+21*16+6*$SIZE_T`
1870 blr
1871 .long 0
1872 .byte 0,12,0x14,0,0x80,6,6,0
1873 .long 0
1874.size .${prefix}_ctr32_encrypt_blocks,.-.${prefix}_ctr32_encrypt_blocks
1875___
1876}} }}}
1877
1878my $consts=1;
1879foreach(split("\n",$code)) {
1880 s/\`([^\`]*)\`/eval($1)/geo;
1881
1882 # constants table endian-specific conversion
1883 if ($consts && m/\.(long|byte)\s+(.+)\s+(\?[a-z]*)$/o) {
1884 my $conv=$3;
1885 my @bytes=();
1886
1887 # convert to endian-agnostic format
1888 if ($1 eq "long") {
1889 foreach (split(/,\s*/,$2)) {
1890 my $l = /^0/?oct:int;
1891 push @bytes,($l>>24)&0xff,($l>>16)&0xff,($l>>8)&0xff,$l&0xff;
1892 }
1893 } else {
1894 @bytes = map(/^0/?oct:int,split(/,\s*/,$2));
1895 }
1896
1897 # little-endian conversion
1898 if ($flavour =~ /le$/o) {
1899 SWITCH: for($conv) {
1900 /\?inv/ && do { @bytes=map($_^0xf,@bytes); last; };
1901 /\?rev/ && do { @bytes=reverse(@bytes); last; };
1902 }
1903 }
1904
1905 #emit
1906 print ".byte\t",join(',',map (sprintf("0x%02x",$_),@bytes)),"\n";
1907 next;
1908 }
1909 $consts=0 if (m/Lconsts:/o); # end of table
1910
1911 # instructions prefixed with '?' are endian-specific and need
1912 # to be adjusted accordingly...
1913 if ($flavour =~ /le$/o) { # little-endian
1914 s/le\?//o or
1915 s/be\?/#be#/o or
1916 s/\?lvsr/lvsl/o or
1917 s/\?lvsl/lvsr/o or
1918 s/\?(vperm\s+v[0-9]+,\s*)(v[0-9]+,\s*)(v[0-9]+,\s*)(v[0-9]+)/$1$3$2$4/o or
1919 s/\?(vsldoi\s+v[0-9]+,\s*)(v[0-9]+,)\s*(v[0-9]+,\s*)([0-9]+)/$1$3$2 16-$4/o or
1920 s/\?(vspltw\s+v[0-9]+,\s*)(v[0-9]+,)\s*([0-9])/$1$2 3-$3/o;
1921 } else { # big-endian
1922 s/le\?/#le#/o or
1923 s/be\?//o or
1924 s/\?([a-z]+)/$1/o;
1925 }
1926
1927 print $_,"\n";
1928}
1929
1930close STDOUT;
diff --git a/drivers/crypto/vmx/ghash.c b/drivers/crypto/vmx/ghash.c
new file mode 100644
index 000000000000..d0ffe277af5c
--- /dev/null
+++ b/drivers/crypto/vmx/ghash.c
@@ -0,0 +1,214 @@
1/**
2 * GHASH routines supporting VMX instructions on the Power 8
3 *
4 * Copyright (C) 2015 International Business Machines Inc.
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; version 2 only.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
18 *
19 * Author: Marcelo Henrique Cerri <mhcerri@br.ibm.com>
20 */
21
22#include <linux/types.h>
23#include <linux/err.h>
24#include <linux/crypto.h>
25#include <linux/delay.h>
26#include <linux/hardirq.h>
27#include <asm/switch_to.h>
28#include <crypto/aes.h>
29#include <crypto/scatterwalk.h>
30#include <crypto/internal/hash.h>
31#include <crypto/b128ops.h>
32
33#define IN_INTERRUPT in_interrupt()
34
35#define GHASH_BLOCK_SIZE (16)
36#define GHASH_DIGEST_SIZE (16)
37#define GHASH_KEY_LEN (16)
38
39void gcm_init_p8(u128 htable[16], const u64 Xi[2]);
40void gcm_gmult_p8(u64 Xi[2], const u128 htable[16]);
41void gcm_ghash_p8(u64 Xi[2], const u128 htable[16],
42 const u8 *in,size_t len);
43
44struct p8_ghash_ctx {
45 u128 htable[16];
46 struct crypto_shash *fallback;
47};
48
49struct p8_ghash_desc_ctx {
50 u64 shash[2];
51 u8 buffer[GHASH_DIGEST_SIZE];
52 int bytes;
53 struct shash_desc fallback_desc;
54};
55
56static int p8_ghash_init_tfm(struct crypto_tfm *tfm)
57{
58 const char *alg;
59 struct crypto_shash *fallback;
60 struct crypto_shash *shash_tfm = __crypto_shash_cast(tfm);
61 struct p8_ghash_ctx *ctx = crypto_tfm_ctx(tfm);
62
63 if (!(alg = crypto_tfm_alg_name(tfm))) {
64 printk(KERN_ERR "Failed to get algorithm name.\n");
65 return -ENOENT;
66 }
67
68 fallback = crypto_alloc_shash(alg, 0 ,CRYPTO_ALG_NEED_FALLBACK);
69 if (IS_ERR(fallback)) {
70 printk(KERN_ERR "Failed to allocate transformation for '%s': %ld\n",
71 alg, PTR_ERR(fallback));
72 return PTR_ERR(fallback);
73 }
74 printk(KERN_INFO "Using '%s' as fallback implementation.\n",
75 crypto_tfm_alg_driver_name(crypto_shash_tfm(fallback)));
76
77 crypto_shash_set_flags(fallback,
78 crypto_shash_get_flags((struct crypto_shash *) tfm));
79 ctx->fallback = fallback;
80
81 shash_tfm->descsize = sizeof(struct p8_ghash_desc_ctx)
82 + crypto_shash_descsize(fallback);
83
84 return 0;
85}
86
87static void p8_ghash_exit_tfm(struct crypto_tfm *tfm)
88{
89 struct p8_ghash_ctx *ctx = crypto_tfm_ctx(tfm);
90
91 if (ctx->fallback) {
92 crypto_free_shash(ctx->fallback);
93 ctx->fallback = NULL;
94 }
95}
96
97static int p8_ghash_init(struct shash_desc *desc)
98{
99 struct p8_ghash_ctx *ctx = crypto_tfm_ctx(crypto_shash_tfm(desc->tfm));
100 struct p8_ghash_desc_ctx *dctx = shash_desc_ctx(desc);
101
102 dctx->bytes = 0;
103 memset(dctx->shash, 0, GHASH_DIGEST_SIZE);
104 dctx->fallback_desc.tfm = ctx->fallback;
105 dctx->fallback_desc.flags = desc->flags;
106 return crypto_shash_init(&dctx->fallback_desc);
107}
108
109static int p8_ghash_setkey(struct crypto_shash *tfm, const u8 *key,
110 unsigned int keylen)
111{
112 struct p8_ghash_ctx *ctx = crypto_tfm_ctx(crypto_shash_tfm(tfm));
113
114 if (keylen != GHASH_KEY_LEN)
115 return -EINVAL;
116
117 pagefault_disable();
118 enable_kernel_altivec();
119 enable_kernel_fp();
120 gcm_init_p8(ctx->htable, (const u64 *) key);
121 pagefault_enable();
122 return crypto_shash_setkey(ctx->fallback, key, keylen);
123}
124
125static int p8_ghash_update(struct shash_desc *desc,
126 const u8 *src, unsigned int srclen)
127{
128 unsigned int len;
129 struct p8_ghash_ctx *ctx = crypto_tfm_ctx(crypto_shash_tfm(desc->tfm));
130 struct p8_ghash_desc_ctx *dctx = shash_desc_ctx(desc);
131
132 if (IN_INTERRUPT) {
133 return crypto_shash_update(&dctx->fallback_desc, src, srclen);
134 } else {
135 if (dctx->bytes) {
136 if (dctx->bytes + srclen < GHASH_DIGEST_SIZE) {
137 memcpy(dctx->buffer + dctx->bytes, src, srclen);
138 dctx->bytes += srclen;
139 return 0;
140 }
141 memcpy(dctx->buffer + dctx->bytes, src,
142 GHASH_DIGEST_SIZE - dctx->bytes);
143 pagefault_disable();
144 enable_kernel_altivec();
145 enable_kernel_fp();
146 gcm_ghash_p8(dctx->shash, ctx->htable, dctx->buffer,
147 GHASH_DIGEST_SIZE);
148 pagefault_enable();
149 src += GHASH_DIGEST_SIZE - dctx->bytes;
150 srclen -= GHASH_DIGEST_SIZE - dctx->bytes;
151 dctx->bytes = 0;
152 }
153 len = srclen & ~(GHASH_DIGEST_SIZE - 1);
154 if (len) {
155 pagefault_disable();
156 enable_kernel_altivec();
157 enable_kernel_fp();
158 gcm_ghash_p8(dctx->shash, ctx->htable, src, len);
159 pagefault_enable();
160 src += len;
161 srclen -= len;
162 }
163 if (srclen) {
164 memcpy(dctx->buffer, src, srclen);
165 dctx->bytes = srclen;
166 }
167 return 0;
168 }
169}
170
171static int p8_ghash_final(struct shash_desc *desc, u8 *out)
172{
173 int i;
174 struct p8_ghash_ctx *ctx = crypto_tfm_ctx(crypto_shash_tfm(desc->tfm));
175 struct p8_ghash_desc_ctx *dctx = shash_desc_ctx(desc);
176
177 if (IN_INTERRUPT) {
178 return crypto_shash_final(&dctx->fallback_desc, out);
179 } else {
180 if (dctx->bytes) {
181 for (i = dctx->bytes; i < GHASH_DIGEST_SIZE; i++)
182 dctx->buffer[i] = 0;
183 pagefault_disable();
184 enable_kernel_altivec();
185 enable_kernel_fp();
186 gcm_ghash_p8(dctx->shash, ctx->htable, dctx->buffer,
187 GHASH_DIGEST_SIZE);
188 pagefault_enable();
189 dctx->bytes = 0;
190 }
191 memcpy(out, dctx->shash, GHASH_DIGEST_SIZE);
192 return 0;
193 }
194}
195
196struct shash_alg p8_ghash_alg = {
197 .digestsize = GHASH_DIGEST_SIZE,
198 .init = p8_ghash_init,
199 .update = p8_ghash_update,
200 .final = p8_ghash_final,
201 .setkey = p8_ghash_setkey,
202 .descsize = sizeof(struct p8_ghash_desc_ctx),
203 .base = {
204 .cra_name = "ghash",
205 .cra_driver_name = "p8_ghash",
206 .cra_priority = 1000,
207 .cra_flags = CRYPTO_ALG_TYPE_SHASH | CRYPTO_ALG_NEED_FALLBACK,
208 .cra_blocksize = GHASH_BLOCK_SIZE,
209 .cra_ctxsize = sizeof(struct p8_ghash_ctx),
210 .cra_module = THIS_MODULE,
211 .cra_init = p8_ghash_init_tfm,
212 .cra_exit = p8_ghash_exit_tfm,
213 },
214};
diff --git a/drivers/crypto/vmx/ghashp8-ppc.pl b/drivers/crypto/vmx/ghashp8-ppc.pl
new file mode 100644
index 000000000000..0a6f899839dd
--- /dev/null
+++ b/drivers/crypto/vmx/ghashp8-ppc.pl
@@ -0,0 +1,228 @@
1#!/usr/bin/env perl
2#
3# ====================================================================
4# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
5# project. The module is, however, dual licensed under OpenSSL and
6# CRYPTOGAMS licenses depending on where you obtain it. For further
7# details see http://www.openssl.org/~appro/cryptogams/.
8# ====================================================================
9#
10# GHASH for for PowerISA v2.07.
11#
12# July 2014
13#
14# Accurate performance measurements are problematic, because it's
15# always virtualized setup with possibly throttled processor.
16# Relative comparison is therefore more informative. This initial
17# version is ~2.1x slower than hardware-assisted AES-128-CTR, ~12x
18# faster than "4-bit" integer-only compiler-generated 64-bit code.
19# "Initial version" means that there is room for futher improvement.
20
21$flavour=shift;
22$output =shift;
23
24if ($flavour =~ /64/) {
25 $SIZE_T=8;
26 $LRSAVE=2*$SIZE_T;
27 $STU="stdu";
28 $POP="ld";
29 $PUSH="std";
30} elsif ($flavour =~ /32/) {
31 $SIZE_T=4;
32 $LRSAVE=$SIZE_T;
33 $STU="stwu";
34 $POP="lwz";
35 $PUSH="stw";
36} else { die "nonsense $flavour"; }
37
38$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
39( $xlate="${dir}ppc-xlate.pl" and -f $xlate ) or
40( $xlate="${dir}../../perlasm/ppc-xlate.pl" and -f $xlate) or
41die "can't locate ppc-xlate.pl";
42
43open STDOUT,"| $^X $xlate $flavour $output" || die "can't call $xlate: $!";
44
45my ($Xip,$Htbl,$inp,$len)=map("r$_",(3..6)); # argument block
46
47my ($Xl,$Xm,$Xh,$IN)=map("v$_",(0..3));
48my ($zero,$t0,$t1,$t2,$xC2,$H,$Hh,$Hl,$lemask)=map("v$_",(4..12));
49my $vrsave="r12";
50
51$code=<<___;
52.machine "any"
53
54.text
55
56.globl .gcm_init_p8
57 lis r0,0xfff0
58 li r8,0x10
59 mfspr $vrsave,256
60 li r9,0x20
61 mtspr 256,r0
62 li r10,0x30
63 lvx_u $H,0,r4 # load H
64
65 vspltisb $xC2,-16 # 0xf0
66 vspltisb $t0,1 # one
67 vaddubm $xC2,$xC2,$xC2 # 0xe0
68 vxor $zero,$zero,$zero
69 vor $xC2,$xC2,$t0 # 0xe1
70 vsldoi $xC2,$xC2,$zero,15 # 0xe1...
71 vsldoi $t1,$zero,$t0,1 # ...1
72 vaddubm $xC2,$xC2,$xC2 # 0xc2...
73 vspltisb $t2,7
74 vor $xC2,$xC2,$t1 # 0xc2....01
75 vspltb $t1,$H,0 # most significant byte
76 vsl $H,$H,$t0 # H<<=1
77 vsrab $t1,$t1,$t2 # broadcast carry bit
78 vand $t1,$t1,$xC2
79 vxor $H,$H,$t1 # twisted H
80
81 vsldoi $H,$H,$H,8 # twist even more ...
82 vsldoi $xC2,$zero,$xC2,8 # 0xc2.0
83 vsldoi $Hl,$zero,$H,8 # ... and split
84 vsldoi $Hh,$H,$zero,8
85
86 stvx_u $xC2,0,r3 # save pre-computed table
87 stvx_u $Hl,r8,r3
88 stvx_u $H, r9,r3
89 stvx_u $Hh,r10,r3
90
91 mtspr 256,$vrsave
92 blr
93 .long 0
94 .byte 0,12,0x14,0,0,0,2,0
95 .long 0
96.size .gcm_init_p8,.-.gcm_init_p8
97
98.globl .gcm_gmult_p8
99 lis r0,0xfff8
100 li r8,0x10
101 mfspr $vrsave,256
102 li r9,0x20
103 mtspr 256,r0
104 li r10,0x30
105 lvx_u $IN,0,$Xip # load Xi
106
107 lvx_u $Hl,r8,$Htbl # load pre-computed table
108 le?lvsl $lemask,r0,r0
109 lvx_u $H, r9,$Htbl
110 le?vspltisb $t0,0x07
111 lvx_u $Hh,r10,$Htbl
112 le?vxor $lemask,$lemask,$t0
113 lvx_u $xC2,0,$Htbl
114 le?vperm $IN,$IN,$IN,$lemask
115 vxor $zero,$zero,$zero
116
117 vpmsumd $Xl,$IN,$Hl # H.lo·Xi.lo
118 vpmsumd $Xm,$IN,$H # H.hi·Xi.lo+H.lo·Xi.hi
119 vpmsumd $Xh,$IN,$Hh # H.hi·Xi.hi
120
121 vpmsumd $t2,$Xl,$xC2 # 1st phase
122
123 vsldoi $t0,$Xm,$zero,8
124 vsldoi $t1,$zero,$Xm,8
125 vxor $Xl,$Xl,$t0
126 vxor $Xh,$Xh,$t1
127
128 vsldoi $Xl,$Xl,$Xl,8
129 vxor $Xl,$Xl,$t2
130
131 vsldoi $t1,$Xl,$Xl,8 # 2nd phase
132 vpmsumd $Xl,$Xl,$xC2
133 vxor $t1,$t1,$Xh
134 vxor $Xl,$Xl,$t1
135
136 le?vperm $Xl,$Xl,$Xl,$lemask
137 stvx_u $Xl,0,$Xip # write out Xi
138
139 mtspr 256,$vrsave
140 blr
141 .long 0
142 .byte 0,12,0x14,0,0,0,2,0
143 .long 0
144.size .gcm_gmult_p8,.-.gcm_gmult_p8
145
146.globl .gcm_ghash_p8
147 lis r0,0xfff8
148 li r8,0x10
149 mfspr $vrsave,256
150 li r9,0x20
151 mtspr 256,r0
152 li r10,0x30
153 lvx_u $Xl,0,$Xip # load Xi
154
155 lvx_u $Hl,r8,$Htbl # load pre-computed table
156 le?lvsl $lemask,r0,r0
157 lvx_u $H, r9,$Htbl
158 le?vspltisb $t0,0x07
159 lvx_u $Hh,r10,$Htbl
160 le?vxor $lemask,$lemask,$t0
161 lvx_u $xC2,0,$Htbl
162 le?vperm $Xl,$Xl,$Xl,$lemask
163 vxor $zero,$zero,$zero
164
165 lvx_u $IN,0,$inp
166 addi $inp,$inp,16
167 subi $len,$len,16
168 le?vperm $IN,$IN,$IN,$lemask
169 vxor $IN,$IN,$Xl
170 b Loop
171
172.align 5
173Loop:
174 subic $len,$len,16
175 vpmsumd $Xl,$IN,$Hl # H.lo·Xi.lo
176 subfe. r0,r0,r0 # borrow?-1:0
177 vpmsumd $Xm,$IN,$H # H.hi·Xi.lo+H.lo·Xi.hi
178 and r0,r0,$len
179 vpmsumd $Xh,$IN,$Hh # H.hi·Xi.hi
180 add $inp,$inp,r0
181
182 vpmsumd $t2,$Xl,$xC2 # 1st phase
183
184 vsldoi $t0,$Xm,$zero,8
185 vsldoi $t1,$zero,$Xm,8
186 vxor $Xl,$Xl,$t0
187 vxor $Xh,$Xh,$t1
188
189 vsldoi $Xl,$Xl,$Xl,8
190 vxor $Xl,$Xl,$t2
191 lvx_u $IN,0,$inp
192 addi $inp,$inp,16
193
194 vsldoi $t1,$Xl,$Xl,8 # 2nd phase
195 vpmsumd $Xl,$Xl,$xC2
196 le?vperm $IN,$IN,$IN,$lemask
197 vxor $t1,$t1,$Xh
198 vxor $IN,$IN,$t1
199 vxor $IN,$IN,$Xl
200 beq Loop # did $len-=16 borrow?
201
202 vxor $Xl,$Xl,$t1
203 le?vperm $Xl,$Xl,$Xl,$lemask
204 stvx_u $Xl,0,$Xip # write out Xi
205
206 mtspr 256,$vrsave
207 blr
208 .long 0
209 .byte 0,12,0x14,0,0,0,4,0
210 .long 0
211.size .gcm_ghash_p8,.-.gcm_ghash_p8
212
213.asciz "GHASH for PowerISA 2.07, CRYPTOGAMS by <appro\@openssl.org>"
214.align 2
215___
216
217foreach (split("\n",$code)) {
218 if ($flavour =~ /le$/o) { # little-endian
219 s/le\?//o or
220 s/be\?/#be#/o;
221 } else {
222 s/le\?/#le#/o or
223 s/be\?//o;
224 }
225 print $_,"\n";
226}
227
228close STDOUT; # enforce flush
diff --git a/drivers/crypto/vmx/ppc-xlate.pl b/drivers/crypto/vmx/ppc-xlate.pl
new file mode 100644
index 000000000000..a59188494af8
--- /dev/null
+++ b/drivers/crypto/vmx/ppc-xlate.pl
@@ -0,0 +1,207 @@
1#!/usr/bin/env perl
2
3# PowerPC assembler distiller by <appro>.
4
5my $flavour = shift;
6my $output = shift;
7open STDOUT,">$output" || die "can't open $output: $!";
8
9my %GLOBALS;
10my $dotinlocallabels=($flavour=~/linux/)?1:0;
11
12################################################################
13# directives which need special treatment on different platforms
14################################################################
15my $globl = sub {
16 my $junk = shift;
17 my $name = shift;
18 my $global = \$GLOBALS{$name};
19 my $ret;
20
21 $name =~ s|^[\.\_]||;
22
23 SWITCH: for ($flavour) {
24 /aix/ && do { $name = ".$name";
25 last;
26 };
27 /osx/ && do { $name = "_$name";
28 last;
29 };
30 /linux/
31 && do { $ret = "_GLOBAL($name)";
32 last;
33 };
34 }
35
36 $ret = ".globl $name\nalign 5\n$name:" if (!$ret);
37 $$global = $name;
38 $ret;
39};
40my $text = sub {
41 my $ret = ($flavour =~ /aix/) ? ".csect\t.text[PR],7" : ".text";
42 $ret = ".abiversion 2\n".$ret if ($flavour =~ /linux.*64le/);
43 $ret;
44};
45my $machine = sub {
46 my $junk = shift;
47 my $arch = shift;
48 if ($flavour =~ /osx/)
49 { $arch =~ s/\"//g;
50 $arch = ($flavour=~/64/) ? "ppc970-64" : "ppc970" if ($arch eq "any");
51 }
52 ".machine $arch";
53};
54my $size = sub {
55 if ($flavour =~ /linux/)
56 { shift;
57 my $name = shift; $name =~ s|^[\.\_]||;
58 my $ret = ".size $name,.-".($flavour=~/64$/?".":"").$name;
59 $ret .= "\n.size .$name,.-.$name" if ($flavour=~/64$/);
60 $ret;
61 }
62 else
63 { ""; }
64};
65my $asciz = sub {
66 shift;
67 my $line = join(",",@_);
68 if ($line =~ /^"(.*)"$/)
69 { ".byte " . join(",",unpack("C*",$1),0) . "\n.align 2"; }
70 else
71 { ""; }
72};
73my $quad = sub {
74 shift;
75 my @ret;
76 my ($hi,$lo);
77 for (@_) {
78 if (/^0x([0-9a-f]*?)([0-9a-f]{1,8})$/io)
79 { $hi=$1?"0x$1":"0"; $lo="0x$2"; }
80 elsif (/^([0-9]+)$/o)
81 { $hi=$1>>32; $lo=$1&0xffffffff; } # error-prone with 32-bit perl
82 else
83 { $hi=undef; $lo=$_; }
84
85 if (defined($hi))
86 { push(@ret,$flavour=~/le$/o?".long\t$lo,$hi":".long\t$hi,$lo"); }
87 else
88 { push(@ret,".quad $lo"); }
89 }
90 join("\n",@ret);
91};
92
93################################################################
94# simplified mnemonics not handled by at least one assembler
95################################################################
96my $cmplw = sub {
97 my $f = shift;
98 my $cr = 0; $cr = shift if ($#_>1);
99 # Some out-of-date 32-bit GNU assembler just can't handle cmplw...
100 ($flavour =~ /linux.*32/) ?
101 " .long ".sprintf "0x%x",31<<26|$cr<<23|$_[0]<<16|$_[1]<<11|64 :
102 " cmplw ".join(',',$cr,@_);
103};
104my $bdnz = sub {
105 my $f = shift;
106 my $bo = $f=~/[\+\-]/ ? 16+9 : 16; # optional "to be taken" hint
107 " bc $bo,0,".shift;
108} if ($flavour!~/linux/);
109my $bltlr = sub {
110 my $f = shift;
111 my $bo = $f=~/\-/ ? 12+2 : 12; # optional "not to be taken" hint
112 ($flavour =~ /linux/) ? # GNU as doesn't allow most recent hints
113 " .long ".sprintf "0x%x",19<<26|$bo<<21|16<<1 :
114 " bclr $bo,0";
115};
116my $bnelr = sub {
117 my $f = shift;
118 my $bo = $f=~/\-/ ? 4+2 : 4; # optional "not to be taken" hint
119 ($flavour =~ /linux/) ? # GNU as doesn't allow most recent hints
120 " .long ".sprintf "0x%x",19<<26|$bo<<21|2<<16|16<<1 :
121 " bclr $bo,2";
122};
123my $beqlr = sub {
124 my $f = shift;
125 my $bo = $f=~/-/ ? 12+2 : 12; # optional "not to be taken" hint
126 ($flavour =~ /linux/) ? # GNU as doesn't allow most recent hints
127 " .long ".sprintf "0x%X",19<<26|$bo<<21|2<<16|16<<1 :
128 " bclr $bo,2";
129};
130# GNU assembler can't handle extrdi rA,rS,16,48, or when sum of last two
131# arguments is 64, with "operand out of range" error.
132my $extrdi = sub {
133 my ($f,$ra,$rs,$n,$b) = @_;
134 $b = ($b+$n)&63; $n = 64-$n;
135 " rldicl $ra,$rs,$b,$n";
136};
137my $vmr = sub {
138 my ($f,$vx,$vy) = @_;
139 " vor $vx,$vy,$vy";
140};
141
142# PowerISA 2.06 stuff
143sub vsxmem_op {
144 my ($f, $vrt, $ra, $rb, $op) = @_;
145 " .long ".sprintf "0x%X",(31<<26)|($vrt<<21)|($ra<<16)|($rb<<11)|($op*2+1);
146}
147# made-up unaligned memory reference AltiVec/VMX instructions
148my $lvx_u = sub { vsxmem_op(@_, 844); }; # lxvd2x
149my $stvx_u = sub { vsxmem_op(@_, 972); }; # stxvd2x
150my $lvdx_u = sub { vsxmem_op(@_, 588); }; # lxsdx
151my $stvdx_u = sub { vsxmem_op(@_, 716); }; # stxsdx
152my $lvx_4w = sub { vsxmem_op(@_, 780); }; # lxvw4x
153my $stvx_4w = sub { vsxmem_op(@_, 908); }; # stxvw4x
154
155# PowerISA 2.07 stuff
156sub vcrypto_op {
157 my ($f, $vrt, $vra, $vrb, $op) = @_;
158 " .long ".sprintf "0x%X",(4<<26)|($vrt<<21)|($vra<<16)|($vrb<<11)|$op;
159}
160my $vcipher = sub { vcrypto_op(@_, 1288); };
161my $vcipherlast = sub { vcrypto_op(@_, 1289); };
162my $vncipher = sub { vcrypto_op(@_, 1352); };
163my $vncipherlast= sub { vcrypto_op(@_, 1353); };
164my $vsbox = sub { vcrypto_op(@_, 0, 1480); };
165my $vshasigmad = sub { my ($st,$six)=splice(@_,-2); vcrypto_op(@_, $st<<4|$six, 1730); };
166my $vshasigmaw = sub { my ($st,$six)=splice(@_,-2); vcrypto_op(@_, $st<<4|$six, 1666); };
167my $vpmsumb = sub { vcrypto_op(@_, 1032); };
168my $vpmsumd = sub { vcrypto_op(@_, 1224); };
169my $vpmsubh = sub { vcrypto_op(@_, 1096); };
170my $vpmsumw = sub { vcrypto_op(@_, 1160); };
171my $vaddudm = sub { vcrypto_op(@_, 192); };
172
173my $mtsle = sub {
174 my ($f, $arg) = @_;
175 " .long ".sprintf "0x%X",(31<<26)|($arg<<21)|(147*2);
176};
177
178print "#include <asm/ppc_asm.h>\n" if $flavour =~ /linux/;
179
180while($line=<>) {
181
182 $line =~ s|[#!;].*$||; # get rid of asm-style comments...
183 $line =~ s|/\*.*\*/||; # ... and C-style comments...
184 $line =~ s|^\s+||; # ... and skip white spaces in beginning...
185 $line =~ s|\s+$||; # ... and at the end
186
187 {
188 $line =~ s|\b\.L(\w+)|L$1|g; # common denominator for Locallabel
189 $line =~ s|\bL(\w+)|\.L$1|g if ($dotinlocallabels);
190 }
191
192 {
193 $line =~ s|^\s*(\.?)(\w+)([\.\+\-]?)\s*||;
194 my $c = $1; $c = "\t" if ($c eq "");
195 my $mnemonic = $2;
196 my $f = $3;
197 my $opcode = eval("\$$mnemonic");
198 $line =~ s/\b(c?[rf]|v|vs)([0-9]+)\b/$2/g if ($c ne "." and $flavour !~ /osx/);
199 if (ref($opcode) eq 'CODE') { $line = &$opcode($f,split(',',$line)); }
200 elsif ($mnemonic) { $line = $c.$mnemonic.$f."\t".$line; }
201 }
202
203 print $line if ($line);
204 print "\n";
205}
206
207close STDOUT;
diff --git a/drivers/crypto/vmx/vmx.c b/drivers/crypto/vmx/vmx.c
new file mode 100644
index 000000000000..44d8d5cfe40d
--- /dev/null
+++ b/drivers/crypto/vmx/vmx.c
@@ -0,0 +1,88 @@
1/**
2 * Routines supporting VMX instructions on the Power 8
3 *
4 * Copyright (C) 2015 International Business Machines Inc.
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; version 2 only.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
18 *
19 * Author: Marcelo Henrique Cerri <mhcerri@br.ibm.com>
20 */
21
22#include <linux/module.h>
23#include <linux/moduleparam.h>
24#include <linux/types.h>
25#include <linux/err.h>
26#include <linux/crypto.h>
27#include <asm/cputable.h>
28#include <crypto/internal/hash.h>
29
30extern struct shash_alg p8_ghash_alg;
31extern struct crypto_alg p8_aes_alg;
32extern struct crypto_alg p8_aes_cbc_alg;
33extern struct crypto_alg p8_aes_ctr_alg;
34static struct crypto_alg *algs[] = {
35 &p8_aes_alg,
36 &p8_aes_cbc_alg,
37 &p8_aes_ctr_alg,
38 NULL,
39};
40
41int __init p8_init(void)
42{
43 int ret = 0;
44 struct crypto_alg **alg_it;
45
46 if (!(cur_cpu_spec->cpu_user_features2 & PPC_FEATURE2_VEC_CRYPTO))
47 return -ENODEV;
48
49 for (alg_it = algs; *alg_it; alg_it++) {
50 ret = crypto_register_alg(*alg_it);
51 printk(KERN_INFO "crypto_register_alg '%s' = %d\n",
52 (*alg_it)->cra_name, ret);
53 if (ret) {
54 for (alg_it--; alg_it >= algs; alg_it--)
55 crypto_unregister_alg(*alg_it);
56 break;
57 }
58 }
59 if (ret)
60 return ret;
61
62 ret = crypto_register_shash(&p8_ghash_alg);
63 if (ret) {
64 for (alg_it = algs; *alg_it; alg_it++)
65 crypto_unregister_alg(*alg_it);
66 }
67 return ret;
68}
69
70void __exit p8_exit(void)
71{
72 struct crypto_alg **alg_it;
73
74 for (alg_it = algs; *alg_it; alg_it++) {
75 printk(KERN_INFO "Removing '%s'\n", (*alg_it)->cra_name);
76 crypto_unregister_alg(*alg_it);
77 }
78 crypto_unregister_shash(&p8_ghash_alg);
79}
80
81module_init(p8_init);
82module_exit(p8_exit);
83
84MODULE_AUTHOR("Marcelo Cerri<mhcerri@br.ibm.com>");
85MODULE_DESCRIPTION("IBM VMX cryptogaphic acceleration instructions support on Power 8");
86MODULE_LICENSE("GPL");
87MODULE_VERSION("1.0.0");
88
generated by cgit v1.2.2 (git 2.25.0) at 2024-11-26 10:18:01 -0500