aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/dma
diff options
context:
space:
mode:
authorVinod Koul <vinod.koul@intel.com>2013-11-16 01:24:17 -0500
committerVinod Koul <vinod.koul@intel.com>2013-11-16 01:32:36 -0500
commitdf12a3178d340319b1955be6b973a4eb84aff754 (patch)
tree2b9c68f8a6c299d1e5a4026c60117b5c00d46008 /drivers/dma
parent2f986ec6fa57a5dcf77f19f5f0d44b1f680a100f (diff)
parent82a1402eaee5dab1f3ab2d5aa4c316451374c5af (diff)
Merge commit 'dmaengine-3.13-v2' of git://git.kernel.org/pub/scm/linux/kernel/git/djbw/dmaengine
Pull dmaengine changes from Dan 1/ Bartlomiej and Dan finalized a rework of the dma address unmap implementation. 2/ In the course of testing 1/ a collection of enhancements to dmatest fell out. Notably basic performance statistics, and fixed / enhanced test control through new module parameters 'run', 'wait', 'noverify', and 'verbose'. Thanks to Andriy and Linus for their review. 3/ Testing the raid related corner cases of 1/ triggered bugs in the recently added 16-source operation support in the ioatdma driver. 4/ Some minor fixes / cleanups to mv_xor and ioatdma. Conflicts: drivers/dma/dmatest.c Signed-off-by: Vinod Koul <vinod.koul@intel.com>
Diffstat (limited to 'drivers/dma')
-rw-r--r--drivers/dma/amba-pl08x.c32
-rw-r--r--drivers/dma/at_hdmac.c26
-rw-r--r--drivers/dma/dmaengine.c262
-rw-r--r--drivers/dma/dmatest.c915
-rw-r--r--drivers/dma/dw/core.c25
-rw-r--r--drivers/dma/ep93xx_dma.c30
-rw-r--r--drivers/dma/fsldma.c17
-rw-r--r--drivers/dma/ioat/dma.c49
-rw-r--r--drivers/dma/ioat/dma.h14
-rw-r--r--drivers/dma/ioat/dma_v2.c2
-rw-r--r--drivers/dma/ioat/dma_v2.h1
-rw-r--r--drivers/dma/ioat/dma_v3.c315
-rw-r--r--drivers/dma/ioat/pci.c20
-rw-r--r--drivers/dma/iop-adma.c97
-rw-r--r--drivers/dma/mv_xor.c52
-rw-r--r--drivers/dma/mv_xor.h25
-rw-r--r--drivers/dma/pl330.c2
-rw-r--r--drivers/dma/ppc4xx/adma.c270
-rw-r--r--drivers/dma/timb_dma.c37
-rw-r--r--drivers/dma/txx9dmac.c25
20 files changed, 624 insertions, 1592 deletions
diff --git a/drivers/dma/amba-pl08x.c b/drivers/dma/amba-pl08x.c
index 4ee6533108f6..f2917e2035ba 100644
--- a/drivers/dma/amba-pl08x.c
+++ b/drivers/dma/amba-pl08x.c
@@ -1164,42 +1164,12 @@ static void pl08x_free_txd(struct pl08x_driver_data *pl08x,
1164 kfree(txd); 1164 kfree(txd);
1165} 1165}
1166 1166
1167static void pl08x_unmap_buffers(struct pl08x_txd *txd)
1168{
1169 struct device *dev = txd->vd.tx.chan->device->dev;
1170 struct pl08x_sg *dsg;
1171
1172 if (!(txd->vd.tx.flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
1173 if (txd->vd.tx.flags & DMA_COMPL_SRC_UNMAP_SINGLE)
1174 list_for_each_entry(dsg, &txd->dsg_list, node)
1175 dma_unmap_single(dev, dsg->src_addr, dsg->len,
1176 DMA_TO_DEVICE);
1177 else {
1178 list_for_each_entry(dsg, &txd->dsg_list, node)
1179 dma_unmap_page(dev, dsg->src_addr, dsg->len,
1180 DMA_TO_DEVICE);
1181 }
1182 }
1183 if (!(txd->vd.tx.flags & DMA_COMPL_SKIP_DEST_UNMAP)) {
1184 if (txd->vd.tx.flags & DMA_COMPL_DEST_UNMAP_SINGLE)
1185 list_for_each_entry(dsg, &txd->dsg_list, node)
1186 dma_unmap_single(dev, dsg->dst_addr, dsg->len,
1187 DMA_FROM_DEVICE);
1188 else
1189 list_for_each_entry(dsg, &txd->dsg_list, node)
1190 dma_unmap_page(dev, dsg->dst_addr, dsg->len,
1191 DMA_FROM_DEVICE);
1192 }
1193}
1194
1195static void pl08x_desc_free(struct virt_dma_desc *vd) 1167static void pl08x_desc_free(struct virt_dma_desc *vd)
1196{ 1168{
1197 struct pl08x_txd *txd = to_pl08x_txd(&vd->tx); 1169 struct pl08x_txd *txd = to_pl08x_txd(&vd->tx);
1198 struct pl08x_dma_chan *plchan = to_pl08x_chan(vd->tx.chan); 1170 struct pl08x_dma_chan *plchan = to_pl08x_chan(vd->tx.chan);
1199 1171
1200 if (!plchan->slave) 1172 dma_descriptor_unmap(txd);
1201 pl08x_unmap_buffers(txd);
1202
1203 if (!txd->done) 1173 if (!txd->done)
1204 pl08x_release_mux(plchan); 1174 pl08x_release_mux(plchan);
1205 1175
diff --git a/drivers/dma/at_hdmac.c b/drivers/dma/at_hdmac.c
index 1ef74579447d..e2c04dc81e2a 100644
--- a/drivers/dma/at_hdmac.c
+++ b/drivers/dma/at_hdmac.c
@@ -344,31 +344,7 @@ atc_chain_complete(struct at_dma_chan *atchan, struct at_desc *desc)
344 /* move myself to free_list */ 344 /* move myself to free_list */
345 list_move(&desc->desc_node, &atchan->free_list); 345 list_move(&desc->desc_node, &atchan->free_list);
346 346
347 /* unmap dma addresses (not on slave channels) */ 347 dma_descriptor_unmap(txd);
348 if (!atchan->chan_common.private) {
349 struct device *parent = chan2parent(&atchan->chan_common);
350 if (!(txd->flags & DMA_COMPL_SKIP_DEST_UNMAP)) {
351 if (txd->flags & DMA_COMPL_DEST_UNMAP_SINGLE)
352 dma_unmap_single(parent,
353 desc->lli.daddr,
354 desc->len, DMA_FROM_DEVICE);
355 else
356 dma_unmap_page(parent,
357 desc->lli.daddr,
358 desc->len, DMA_FROM_DEVICE);
359 }
360 if (!(txd->flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
361 if (txd->flags & DMA_COMPL_SRC_UNMAP_SINGLE)
362 dma_unmap_single(parent,
363 desc->lli.saddr,
364 desc->len, DMA_TO_DEVICE);
365 else
366 dma_unmap_page(parent,
367 desc->lli.saddr,
368 desc->len, DMA_TO_DEVICE);
369 }
370 }
371
372 /* for cyclic transfers, 348 /* for cyclic transfers,
373 * no need to replay callback function while stopping */ 349 * no need to replay callback function while stopping */
374 if (!atc_chan_is_cyclic(atchan)) { 350 if (!atc_chan_is_cyclic(atchan)) {
diff --git a/drivers/dma/dmaengine.c b/drivers/dma/dmaengine.c
index 81d876528c70..ea806bdc12ef 100644
--- a/drivers/dma/dmaengine.c
+++ b/drivers/dma/dmaengine.c
@@ -65,6 +65,7 @@
65#include <linux/acpi.h> 65#include <linux/acpi.h>
66#include <linux/acpi_dma.h> 66#include <linux/acpi_dma.h>
67#include <linux/of_dma.h> 67#include <linux/of_dma.h>
68#include <linux/mempool.h>
68 69
69static DEFINE_MUTEX(dma_list_mutex); 70static DEFINE_MUTEX(dma_list_mutex);
70static DEFINE_IDR(dma_idr); 71static DEFINE_IDR(dma_idr);
@@ -901,98 +902,132 @@ void dma_async_device_unregister(struct dma_device *device)
901} 902}
902EXPORT_SYMBOL(dma_async_device_unregister); 903EXPORT_SYMBOL(dma_async_device_unregister);
903 904
904/** 905struct dmaengine_unmap_pool {
905 * dma_async_memcpy_buf_to_buf - offloaded copy between virtual addresses 906 struct kmem_cache *cache;
906 * @chan: DMA channel to offload copy to 907 const char *name;
907 * @dest: destination address (virtual) 908 mempool_t *pool;
908 * @src: source address (virtual) 909 size_t size;
909 * @len: length 910};
910 *
911 * Both @dest and @src must be mappable to a bus address according to the
912 * DMA mapping API rules for streaming mappings.
913 * Both @dest and @src must stay memory resident (kernel memory or locked
914 * user space pages).
915 */
916dma_cookie_t
917dma_async_memcpy_buf_to_buf(struct dma_chan *chan, void *dest,
918 void *src, size_t len)
919{
920 struct dma_device *dev = chan->device;
921 struct dma_async_tx_descriptor *tx;
922 dma_addr_t dma_dest, dma_src;
923 dma_cookie_t cookie;
924 unsigned long flags;
925 911
926 dma_src = dma_map_single(dev->dev, src, len, DMA_TO_DEVICE); 912#define __UNMAP_POOL(x) { .size = x, .name = "dmaengine-unmap-" __stringify(x) }
927 dma_dest = dma_map_single(dev->dev, dest, len, DMA_FROM_DEVICE); 913static struct dmaengine_unmap_pool unmap_pool[] = {
928 flags = DMA_CTRL_ACK | 914 __UNMAP_POOL(2),
929 DMA_COMPL_SRC_UNMAP_SINGLE | 915 #if IS_ENABLED(CONFIG_ASYNC_TX_DMA)
930 DMA_COMPL_DEST_UNMAP_SINGLE; 916 __UNMAP_POOL(16),
931 tx = dev->device_prep_dma_memcpy(chan, dma_dest, dma_src, len, flags); 917 __UNMAP_POOL(128),
918 __UNMAP_POOL(256),
919 #endif
920};
932 921
933 if (!tx) { 922static struct dmaengine_unmap_pool *__get_unmap_pool(int nr)
934 dma_unmap_single(dev->dev, dma_src, len, DMA_TO_DEVICE); 923{
935 dma_unmap_single(dev->dev, dma_dest, len, DMA_FROM_DEVICE); 924 int order = get_count_order(nr);
936 return -ENOMEM; 925
926 switch (order) {
927 case 0 ... 1:
928 return &unmap_pool[0];
929 case 2 ... 4:
930 return &unmap_pool[1];
931 case 5 ... 7:
932 return &unmap_pool[2];
933 case 8:
934 return &unmap_pool[3];
935 default:
936 BUG();
937 return NULL;
937 } 938 }
939}
938 940
939 tx->callback = NULL; 941static void dmaengine_unmap(struct kref *kref)
940 cookie = tx->tx_submit(tx); 942{
943 struct dmaengine_unmap_data *unmap = container_of(kref, typeof(*unmap), kref);
944 struct device *dev = unmap->dev;
945 int cnt, i;
946
947 cnt = unmap->to_cnt;
948 for (i = 0; i < cnt; i++)
949 dma_unmap_page(dev, unmap->addr[i], unmap->len,
950 DMA_TO_DEVICE);
951 cnt += unmap->from_cnt;
952 for (; i < cnt; i++)
953 dma_unmap_page(dev, unmap->addr[i], unmap->len,
954 DMA_FROM_DEVICE);
955 cnt += unmap->bidi_cnt;
956 for (; i < cnt; i++) {
957 if (unmap->addr[i] == 0)
958 continue;
959 dma_unmap_page(dev, unmap->addr[i], unmap->len,
960 DMA_BIDIRECTIONAL);
961 }
962 mempool_free(unmap, __get_unmap_pool(cnt)->pool);
963}
941 964
942 preempt_disable(); 965void dmaengine_unmap_put(struct dmaengine_unmap_data *unmap)
943 __this_cpu_add(chan->local->bytes_transferred, len); 966{
944 __this_cpu_inc(chan->local->memcpy_count); 967 if (unmap)
945 preempt_enable(); 968 kref_put(&unmap->kref, dmaengine_unmap);
969}
970EXPORT_SYMBOL_GPL(dmaengine_unmap_put);
946 971
947 return cookie; 972static void dmaengine_destroy_unmap_pool(void)
973{
974 int i;
975
976 for (i = 0; i < ARRAY_SIZE(unmap_pool); i++) {
977 struct dmaengine_unmap_pool *p = &unmap_pool[i];
978
979 if (p->pool)
980 mempool_destroy(p->pool);
981 p->pool = NULL;
982 if (p->cache)
983 kmem_cache_destroy(p->cache);
984 p->cache = NULL;
985 }
948} 986}
949EXPORT_SYMBOL(dma_async_memcpy_buf_to_buf);
950 987
951/** 988static int __init dmaengine_init_unmap_pool(void)
952 * dma_async_memcpy_buf_to_pg - offloaded copy from address to page
953 * @chan: DMA channel to offload copy to
954 * @page: destination page
955 * @offset: offset in page to copy to
956 * @kdata: source address (virtual)
957 * @len: length
958 *
959 * Both @page/@offset and @kdata must be mappable to a bus address according
960 * to the DMA mapping API rules for streaming mappings.
961 * Both @page/@offset and @kdata must stay memory resident (kernel memory or
962 * locked user space pages)
963 */
964dma_cookie_t
965dma_async_memcpy_buf_to_pg(struct dma_chan *chan, struct page *page,
966 unsigned int offset, void *kdata, size_t len)
967{ 989{
968 struct dma_device *dev = chan->device; 990 int i;
969 struct dma_async_tx_descriptor *tx;
970 dma_addr_t dma_dest, dma_src;
971 dma_cookie_t cookie;
972 unsigned long flags;
973 991
974 dma_src = dma_map_single(dev->dev, kdata, len, DMA_TO_DEVICE); 992 for (i = 0; i < ARRAY_SIZE(unmap_pool); i++) {
975 dma_dest = dma_map_page(dev->dev, page, offset, len, DMA_FROM_DEVICE); 993 struct dmaengine_unmap_pool *p = &unmap_pool[i];
976 flags = DMA_CTRL_ACK | DMA_COMPL_SRC_UNMAP_SINGLE; 994 size_t size;
977 tx = dev->device_prep_dma_memcpy(chan, dma_dest, dma_src, len, flags);
978 995
979 if (!tx) { 996 size = sizeof(struct dmaengine_unmap_data) +
980 dma_unmap_single(dev->dev, dma_src, len, DMA_TO_DEVICE); 997 sizeof(dma_addr_t) * p->size;
981 dma_unmap_page(dev->dev, dma_dest, len, DMA_FROM_DEVICE); 998
982 return -ENOMEM; 999 p->cache = kmem_cache_create(p->name, size, 0,
1000 SLAB_HWCACHE_ALIGN, NULL);
1001 if (!p->cache)
1002 break;
1003 p->pool = mempool_create_slab_pool(1, p->cache);
1004 if (!p->pool)
1005 break;
983 } 1006 }
984 1007
985 tx->callback = NULL; 1008 if (i == ARRAY_SIZE(unmap_pool))
986 cookie = tx->tx_submit(tx); 1009 return 0;
987 1010
988 preempt_disable(); 1011 dmaengine_destroy_unmap_pool();
989 __this_cpu_add(chan->local->bytes_transferred, len); 1012 return -ENOMEM;
990 __this_cpu_inc(chan->local->memcpy_count); 1013}
991 preempt_enable();
992 1014
993 return cookie; 1015struct dmaengine_unmap_data *
1016dmaengine_get_unmap_data(struct device *dev, int nr, gfp_t flags)
1017{
1018 struct dmaengine_unmap_data *unmap;
1019
1020 unmap = mempool_alloc(__get_unmap_pool(nr)->pool, flags);
1021 if (!unmap)
1022 return NULL;
1023
1024 memset(unmap, 0, sizeof(*unmap));
1025 kref_init(&unmap->kref);
1026 unmap->dev = dev;
1027
1028 return unmap;
994} 1029}
995EXPORT_SYMBOL(dma_async_memcpy_buf_to_pg); 1030EXPORT_SYMBOL(dmaengine_get_unmap_data);
996 1031
997/** 1032/**
998 * dma_async_memcpy_pg_to_pg - offloaded copy from page to page 1033 * dma_async_memcpy_pg_to_pg - offloaded copy from page to page
@@ -1015,24 +1050,33 @@ dma_async_memcpy_pg_to_pg(struct dma_chan *chan, struct page *dest_pg,
1015{ 1050{
1016 struct dma_device *dev = chan->device; 1051 struct dma_device *dev = chan->device;
1017 struct dma_async_tx_descriptor *tx; 1052 struct dma_async_tx_descriptor *tx;
1018 dma_addr_t dma_dest, dma_src; 1053 struct dmaengine_unmap_data *unmap;
1019 dma_cookie_t cookie; 1054 dma_cookie_t cookie;
1020 unsigned long flags; 1055 unsigned long flags;
1021 1056
1022 dma_src = dma_map_page(dev->dev, src_pg, src_off, len, DMA_TO_DEVICE); 1057 unmap = dmaengine_get_unmap_data(dev->dev, 2, GFP_NOIO);
1023 dma_dest = dma_map_page(dev->dev, dest_pg, dest_off, len, 1058 if (!unmap)
1024 DMA_FROM_DEVICE); 1059 return -ENOMEM;
1060
1061 unmap->to_cnt = 1;
1062 unmap->from_cnt = 1;
1063 unmap->addr[0] = dma_map_page(dev->dev, src_pg, src_off, len,
1064 DMA_TO_DEVICE);
1065 unmap->addr[1] = dma_map_page(dev->dev, dest_pg, dest_off, len,
1066 DMA_FROM_DEVICE);
1067 unmap->len = len;
1025 flags = DMA_CTRL_ACK; 1068 flags = DMA_CTRL_ACK;
1026 tx = dev->device_prep_dma_memcpy(chan, dma_dest, dma_src, len, flags); 1069 tx = dev->device_prep_dma_memcpy(chan, unmap->addr[1], unmap->addr[0],
1070 len, flags);
1027 1071
1028 if (!tx) { 1072 if (!tx) {
1029 dma_unmap_page(dev->dev, dma_src, len, DMA_TO_DEVICE); 1073 dmaengine_unmap_put(unmap);
1030 dma_unmap_page(dev->dev, dma_dest, len, DMA_FROM_DEVICE);
1031 return -ENOMEM; 1074 return -ENOMEM;
1032 } 1075 }
1033 1076
1034 tx->callback = NULL; 1077 dma_set_unmap(tx, unmap);
1035 cookie = tx->tx_submit(tx); 1078 cookie = tx->tx_submit(tx);
1079 dmaengine_unmap_put(unmap);
1036 1080
1037 preempt_disable(); 1081 preempt_disable();
1038 __this_cpu_add(chan->local->bytes_transferred, len); 1082 __this_cpu_add(chan->local->bytes_transferred, len);
@@ -1043,6 +1087,52 @@ dma_async_memcpy_pg_to_pg(struct dma_chan *chan, struct page *dest_pg,
1043} 1087}
1044EXPORT_SYMBOL(dma_async_memcpy_pg_to_pg); 1088EXPORT_SYMBOL(dma_async_memcpy_pg_to_pg);
1045 1089
1090/**
1091 * dma_async_memcpy_buf_to_buf - offloaded copy between virtual addresses
1092 * @chan: DMA channel to offload copy to
1093 * @dest: destination address (virtual)
1094 * @src: source address (virtual)
1095 * @len: length
1096 *
1097 * Both @dest and @src must be mappable to a bus address according to the
1098 * DMA mapping API rules for streaming mappings.
1099 * Both @dest and @src must stay memory resident (kernel memory or locked
1100 * user space pages).
1101 */
1102dma_cookie_t
1103dma_async_memcpy_buf_to_buf(struct dma_chan *chan, void *dest,
1104 void *src, size_t len)
1105{
1106 return dma_async_memcpy_pg_to_pg(chan, virt_to_page(dest),
1107 (unsigned long) dest & ~PAGE_MASK,
1108 virt_to_page(src),
1109 (unsigned long) src & ~PAGE_MASK, len);
1110}
1111EXPORT_SYMBOL(dma_async_memcpy_buf_to_buf);
1112
1113/**
1114 * dma_async_memcpy_buf_to_pg - offloaded copy from address to page
1115 * @chan: DMA channel to offload copy to
1116 * @page: destination page
1117 * @offset: offset in page to copy to
1118 * @kdata: source address (virtual)
1119 * @len: length
1120 *
1121 * Both @page/@offset and @kdata must be mappable to a bus address according
1122 * to the DMA mapping API rules for streaming mappings.
1123 * Both @page/@offset and @kdata must stay memory resident (kernel memory or
1124 * locked user space pages)
1125 */
1126dma_cookie_t
1127dma_async_memcpy_buf_to_pg(struct dma_chan *chan, struct page *page,
1128 unsigned int offset, void *kdata, size_t len)
1129{
1130 return dma_async_memcpy_pg_to_pg(chan, page, offset,
1131 virt_to_page(kdata),
1132 (unsigned long) kdata & ~PAGE_MASK, len);
1133}
1134EXPORT_SYMBOL(dma_async_memcpy_buf_to_pg);
1135
1046void dma_async_tx_descriptor_init(struct dma_async_tx_descriptor *tx, 1136void dma_async_tx_descriptor_init(struct dma_async_tx_descriptor *tx,
1047 struct dma_chan *chan) 1137 struct dma_chan *chan)
1048{ 1138{
@@ -1116,6 +1206,10 @@ EXPORT_SYMBOL_GPL(dma_run_dependencies);
1116 1206
1117static int __init dma_bus_init(void) 1207static int __init dma_bus_init(void)
1118{ 1208{
1209 int err = dmaengine_init_unmap_pool();
1210
1211 if (err)
1212 return err;
1119 return class_register(&dma_devclass); 1213 return class_register(&dma_devclass);
1120} 1214}
1121arch_initcall(dma_bus_init); 1215arch_initcall(dma_bus_init);
diff --git a/drivers/dma/dmatest.c b/drivers/dma/dmatest.c
index 59e287f56dfc..20f9a3aaf926 100644
--- a/drivers/dma/dmatest.c
+++ b/drivers/dma/dmatest.c
@@ -8,6 +8,8 @@
8 * it under the terms of the GNU General Public License version 2 as 8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation. 9 * published by the Free Software Foundation.
10 */ 10 */
11#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
12
11#include <linux/delay.h> 13#include <linux/delay.h>
12#include <linux/dma-mapping.h> 14#include <linux/dma-mapping.h>
13#include <linux/dmaengine.h> 15#include <linux/dmaengine.h>
@@ -19,10 +21,6 @@
19#include <linux/random.h> 21#include <linux/random.h>
20#include <linux/slab.h> 22#include <linux/slab.h>
21#include <linux/wait.h> 23#include <linux/wait.h>
22#include <linux/ctype.h>
23#include <linux/debugfs.h>
24#include <linux/uaccess.h>
25#include <linux/seq_file.h>
26 24
27static unsigned int test_buf_size = 16384; 25static unsigned int test_buf_size = 16384;
28module_param(test_buf_size, uint, S_IRUGO | S_IWUSR); 26module_param(test_buf_size, uint, S_IRUGO | S_IWUSR);
@@ -68,92 +66,13 @@ module_param(timeout, uint, S_IRUGO | S_IWUSR);
68MODULE_PARM_DESC(timeout, "Transfer Timeout in msec (default: 3000), " 66MODULE_PARM_DESC(timeout, "Transfer Timeout in msec (default: 3000), "
69 "Pass -1 for infinite timeout"); 67 "Pass -1 for infinite timeout");
70 68
71/* Maximum amount of mismatched bytes in buffer to print */ 69static bool noverify;
72#define MAX_ERROR_COUNT 32 70module_param(noverify, bool, S_IRUGO | S_IWUSR);
73 71MODULE_PARM_DESC(noverify, "Disable random data setup and verification");
74/*
75 * Initialization patterns. All bytes in the source buffer has bit 7
76 * set, all bytes in the destination buffer has bit 7 cleared.
77 *
78 * Bit 6 is set for all bytes which are to be copied by the DMA
79 * engine. Bit 5 is set for all bytes which are to be overwritten by
80 * the DMA engine.
81 *
82 * The remaining bits are the inverse of a counter which increments by
83 * one for each byte address.
84 */
85#define PATTERN_SRC 0x80
86#define PATTERN_DST 0x00
87#define PATTERN_COPY 0x40
88#define PATTERN_OVERWRITE 0x20
89#define PATTERN_COUNT_MASK 0x1f
90
91enum dmatest_error_type {
92 DMATEST_ET_OK,
93 DMATEST_ET_MAP_SRC,
94 DMATEST_ET_MAP_DST,
95 DMATEST_ET_PREP,
96 DMATEST_ET_SUBMIT,
97 DMATEST_ET_TIMEOUT,
98 DMATEST_ET_DMA_ERROR,
99 DMATEST_ET_DMA_IN_PROGRESS,
100 DMATEST_ET_VERIFY,
101 DMATEST_ET_VERIFY_BUF,
102};
103
104struct dmatest_verify_buffer {
105 unsigned int index;
106 u8 expected;
107 u8 actual;
108};
109
110struct dmatest_verify_result {
111 unsigned int error_count;
112 struct dmatest_verify_buffer data[MAX_ERROR_COUNT];
113 u8 pattern;
114 bool is_srcbuf;
115};
116
117struct dmatest_thread_result {
118 struct list_head node;
119 unsigned int n;
120 unsigned int src_off;
121 unsigned int dst_off;
122 unsigned int len;
123 enum dmatest_error_type type;
124 union {
125 unsigned long data;
126 dma_cookie_t cookie;
127 enum dma_status status;
128 int error;
129 struct dmatest_verify_result *vr;
130 };
131};
132
133struct dmatest_result {
134 struct list_head node;
135 char *name;
136 struct list_head results;
137};
138
139struct dmatest_info;
140
141struct dmatest_thread {
142 struct list_head node;
143 struct dmatest_info *info;
144 struct task_struct *task;
145 struct dma_chan *chan;
146 u8 **srcs;
147 u8 **dsts;
148 enum dma_transaction_type type;
149 bool done;
150};
151 72
152struct dmatest_chan { 73static bool verbose;
153 struct list_head node; 74module_param(verbose, bool, S_IRUGO | S_IWUSR);
154 struct dma_chan *chan; 75MODULE_PARM_DESC(verbose, "Enable \"success\" result messages (default: off)");
155 struct list_head threads;
156};
157 76
158/** 77/**
159 * struct dmatest_params - test parameters. 78 * struct dmatest_params - test parameters.
@@ -177,6 +96,7 @@ struct dmatest_params {
177 unsigned int xor_sources; 96 unsigned int xor_sources;
178 unsigned int pq_sources; 97 unsigned int pq_sources;
179 int timeout; 98 int timeout;
99 bool noverify;
180}; 100};
181 101
182/** 102/**
@@ -184,7 +104,7 @@ struct dmatest_params {
184 * @params: test parameters 104 * @params: test parameters
185 * @lock: access protection to the fields of this structure 105 * @lock: access protection to the fields of this structure
186 */ 106 */
187struct dmatest_info { 107static struct dmatest_info {
188 /* Test parameters */ 108 /* Test parameters */
189 struct dmatest_params params; 109 struct dmatest_params params;
190 110
@@ -192,16 +112,95 @@ struct dmatest_info {
192 struct list_head channels; 112 struct list_head channels;
193 unsigned int nr_channels; 113 unsigned int nr_channels;
194 struct mutex lock; 114 struct mutex lock;
115 bool did_init;
116} test_info = {
117 .channels = LIST_HEAD_INIT(test_info.channels),
118 .lock = __MUTEX_INITIALIZER(test_info.lock),
119};
120
121static int dmatest_run_set(const char *val, const struct kernel_param *kp);
122static int dmatest_run_get(char *val, const struct kernel_param *kp);
123static struct kernel_param_ops run_ops = {
124 .set = dmatest_run_set,
125 .get = dmatest_run_get,
126};
127static bool dmatest_run;
128module_param_cb(run, &run_ops, &dmatest_run, S_IRUGO | S_IWUSR);
129MODULE_PARM_DESC(run, "Run the test (default: false)");
130
131/* Maximum amount of mismatched bytes in buffer to print */
132#define MAX_ERROR_COUNT 32
133
134/*
135 * Initialization patterns. All bytes in the source buffer has bit 7
136 * set, all bytes in the destination buffer has bit 7 cleared.
137 *
138 * Bit 6 is set for all bytes which are to be copied by the DMA
139 * engine. Bit 5 is set for all bytes which are to be overwritten by
140 * the DMA engine.
141 *
142 * The remaining bits are the inverse of a counter which increments by
143 * one for each byte address.
144 */
145#define PATTERN_SRC 0x80
146#define PATTERN_DST 0x00
147#define PATTERN_COPY 0x40
148#define PATTERN_OVERWRITE 0x20
149#define PATTERN_COUNT_MASK 0x1f
195 150
196 /* debugfs related stuff */ 151struct dmatest_thread {
197 struct dentry *root; 152 struct list_head node;
153 struct dmatest_info *info;
154 struct task_struct *task;
155 struct dma_chan *chan;
156 u8 **srcs;
157 u8 **dsts;
158 enum dma_transaction_type type;
159 bool done;
160};
198 161
199 /* Test results */ 162struct dmatest_chan {
200 struct list_head results; 163 struct list_head node;
201 struct mutex results_lock; 164 struct dma_chan *chan;
165 struct list_head threads;
202}; 166};
203 167
204static struct dmatest_info test_info; 168static DECLARE_WAIT_QUEUE_HEAD(thread_wait);
169static bool wait;
170
171static bool is_threaded_test_run(struct dmatest_info *info)
172{
173 struct dmatest_chan *dtc;
174
175 list_for_each_entry(dtc, &info->channels, node) {
176 struct dmatest_thread *thread;
177
178 list_for_each_entry(thread, &dtc->threads, node) {
179 if (!thread->done)
180 return true;
181 }
182 }
183
184 return false;
185}
186
187static int dmatest_wait_get(char *val, const struct kernel_param *kp)
188{
189 struct dmatest_info *info = &test_info;
190 struct dmatest_params *params = &info->params;
191
192 if (params->iterations)
193 wait_event(thread_wait, !is_threaded_test_run(info));
194 wait = true;
195 return param_get_bool(val, kp);
196}
197
198static struct kernel_param_ops wait_ops = {
199 .get = dmatest_wait_get,
200 .set = param_set_bool,
201};
202module_param_cb(wait, &wait_ops, &wait, S_IRUGO);
203MODULE_PARM_DESC(wait, "Wait for tests to complete (default: false)");
205 204
206static bool dmatest_match_channel(struct dmatest_params *params, 205static bool dmatest_match_channel(struct dmatest_params *params,
207 struct dma_chan *chan) 206 struct dma_chan *chan)
@@ -223,7 +222,7 @@ static unsigned long dmatest_random(void)
223{ 222{
224 unsigned long buf; 223 unsigned long buf;
225 224
226 get_random_bytes(&buf, sizeof(buf)); 225 prandom_bytes(&buf, sizeof(buf));
227 return buf; 226 return buf;
228} 227}
229 228
@@ -262,9 +261,31 @@ static void dmatest_init_dsts(u8 **bufs, unsigned int start, unsigned int len,
262 } 261 }
263} 262}
264 263
265static unsigned int dmatest_verify(struct dmatest_verify_result *vr, u8 **bufs, 264static void dmatest_mismatch(u8 actual, u8 pattern, unsigned int index,
266 unsigned int start, unsigned int end, unsigned int counter, 265 unsigned int counter, bool is_srcbuf)
267 u8 pattern, bool is_srcbuf) 266{
267 u8 diff = actual ^ pattern;
268 u8 expected = pattern | (~counter & PATTERN_COUNT_MASK);
269 const char *thread_name = current->comm;
270
271 if (is_srcbuf)
272 pr_warn("%s: srcbuf[0x%x] overwritten! Expected %02x, got %02x\n",
273 thread_name, index, expected, actual);
274 else if ((pattern & PATTERN_COPY)
275 && (diff & (PATTERN_COPY | PATTERN_OVERWRITE)))
276 pr_warn("%s: dstbuf[0x%x] not copied! Expected %02x, got %02x\n",
277 thread_name, index, expected, actual);
278 else if (diff & PATTERN_SRC)
279 pr_warn("%s: dstbuf[0x%x] was copied! Expected %02x, got %02x\n",
280 thread_name, index, expected, actual);
281 else
282 pr_warn("%s: dstbuf[0x%x] mismatch! Expected %02x, got %02x\n",
283 thread_name, index, expected, actual);
284}
285
286static unsigned int dmatest_verify(u8 **bufs, unsigned int start,
287 unsigned int end, unsigned int counter, u8 pattern,
288 bool is_srcbuf)
268{ 289{
269 unsigned int i; 290 unsigned int i;
270 unsigned int error_count = 0; 291 unsigned int error_count = 0;
@@ -272,7 +293,6 @@ static unsigned int dmatest_verify(struct dmatest_verify_result *vr, u8 **bufs,
272 u8 expected; 293 u8 expected;
273 u8 *buf; 294 u8 *buf;
274 unsigned int counter_orig = counter; 295 unsigned int counter_orig = counter;
275 struct dmatest_verify_buffer *vb;
276 296
277 for (; (buf = *bufs); bufs++) { 297 for (; (buf = *bufs); bufs++) {
278 counter = counter_orig; 298 counter = counter_orig;
@@ -280,12 +300,9 @@ static unsigned int dmatest_verify(struct dmatest_verify_result *vr, u8 **bufs,
280 actual = buf[i]; 300 actual = buf[i];
281 expected = pattern | (~counter & PATTERN_COUNT_MASK); 301 expected = pattern | (~counter & PATTERN_COUNT_MASK);
282 if (actual != expected) { 302 if (actual != expected) {
283 if (error_count < MAX_ERROR_COUNT && vr) { 303 if (error_count < MAX_ERROR_COUNT)
284 vb = &vr->data[error_count]; 304 dmatest_mismatch(actual, pattern, i,
285 vb->index = i; 305 counter, is_srcbuf);
286 vb->expected = expected;
287 vb->actual = actual;
288 }
289 error_count++; 306 error_count++;
290 } 307 }
291 counter++; 308 counter++;
@@ -293,7 +310,7 @@ static unsigned int dmatest_verify(struct dmatest_verify_result *vr, u8 **bufs,
293 } 310 }
294 311
295 if (error_count > MAX_ERROR_COUNT) 312 if (error_count > MAX_ERROR_COUNT)
296 pr_warning("%s: %u errors suppressed\n", 313 pr_warn("%s: %u errors suppressed\n",
297 current->comm, error_count - MAX_ERROR_COUNT); 314 current->comm, error_count - MAX_ERROR_COUNT);
298 315
299 return error_count; 316 return error_count;
@@ -313,20 +330,6 @@ static void dmatest_callback(void *arg)
313 wake_up_all(done->wait); 330 wake_up_all(done->wait);
314} 331}
315 332
316static inline void unmap_src(struct device *dev, dma_addr_t *addr, size_t len,
317 unsigned int count)
318{
319 while (count--)
320 dma_unmap_single(dev, addr[count], len, DMA_TO_DEVICE);
321}
322
323static inline void unmap_dst(struct device *dev, dma_addr_t *addr, size_t len,
324 unsigned int count)
325{
326 while (count--)
327 dma_unmap_single(dev, addr[count], len, DMA_BIDIRECTIONAL);
328}
329
330static unsigned int min_odd(unsigned int x, unsigned int y) 333static unsigned int min_odd(unsigned int x, unsigned int y)
331{ 334{
332 unsigned int val = min(x, y); 335 unsigned int val = min(x, y);
@@ -334,172 +337,49 @@ static unsigned int min_odd(unsigned int x, unsigned int y)
334 return val % 2 ? val : val - 1; 337 return val % 2 ? val : val - 1;
335} 338}
336 339
337static char *verify_result_get_one(struct dmatest_verify_result *vr, 340static void result(const char *err, unsigned int n, unsigned int src_off,
338 unsigned int i) 341 unsigned int dst_off, unsigned int len, unsigned long data)
339{ 342{
340 struct dmatest_verify_buffer *vb = &vr->data[i]; 343 pr_info("%s: result #%u: '%s' with src_off=0x%x dst_off=0x%x len=0x%x (%lu)",
341 u8 diff = vb->actual ^ vr->pattern; 344 current->comm, n, err, src_off, dst_off, len, data);
342 static char buf[512];
343 char *msg;
344
345 if (vr->is_srcbuf)
346 msg = "srcbuf overwritten!";
347 else if ((vr->pattern & PATTERN_COPY)
348 && (diff & (PATTERN_COPY | PATTERN_OVERWRITE)))
349 msg = "dstbuf not copied!";
350 else if (diff & PATTERN_SRC)
351 msg = "dstbuf was copied!";
352 else
353 msg = "dstbuf mismatch!";
354
355 snprintf(buf, sizeof(buf) - 1, "%s [0x%x] Expected %02x, got %02x", msg,
356 vb->index, vb->expected, vb->actual);
357
358 return buf;
359} 345}
360 346
361static char *thread_result_get(const char *name, 347static void dbg_result(const char *err, unsigned int n, unsigned int src_off,
362 struct dmatest_thread_result *tr) 348 unsigned int dst_off, unsigned int len,
349 unsigned long data)
363{ 350{
364 static const char * const messages[] = { 351 pr_debug("%s: result #%u: '%s' with src_off=0x%x dst_off=0x%x len=0x%x (%lu)",
365 [DMATEST_ET_OK] = "No errors", 352 current->comm, n, err, src_off, dst_off, len, data);
366 [DMATEST_ET_MAP_SRC] = "src mapping error",
367 [DMATEST_ET_MAP_DST] = "dst mapping error",
368 [DMATEST_ET_PREP] = "prep error",
369 [DMATEST_ET_SUBMIT] = "submit error",
370 [DMATEST_ET_TIMEOUT] = "test timed out",
371 [DMATEST_ET_DMA_ERROR] =
372 "got completion callback (DMA_ERROR)",
373 [DMATEST_ET_DMA_IN_PROGRESS] =
374 "got completion callback (DMA_IN_PROGRESS)",
375 [DMATEST_ET_VERIFY] = "errors",
376 [DMATEST_ET_VERIFY_BUF] = "verify errors",
377 };
378 static char buf[512];
379
380 snprintf(buf, sizeof(buf) - 1,
381 "%s: #%u: %s with src_off=0x%x ""dst_off=0x%x len=0x%x (%lu)",
382 name, tr->n, messages[tr->type], tr->src_off, tr->dst_off,
383 tr->len, tr->data);
384
385 return buf;
386} 353}
387 354
388static int thread_result_add(struct dmatest_info *info, 355#define verbose_result(err, n, src_off, dst_off, len, data) ({ \
389 struct dmatest_result *r, enum dmatest_error_type type, 356 if (verbose) \
390 unsigned int n, unsigned int src_off, unsigned int dst_off, 357 result(err, n, src_off, dst_off, len, data); \
391 unsigned int len, unsigned long data) 358 else \
392{ 359 dbg_result(err, n, src_off, dst_off, len, data); \
393 struct dmatest_thread_result *tr; 360})
394
395 tr = kzalloc(sizeof(*tr), GFP_KERNEL);
396 if (!tr)
397 return -ENOMEM;
398
399 tr->type = type;
400 tr->n = n;
401 tr->src_off = src_off;
402 tr->dst_off = dst_off;
403 tr->len = len;
404 tr->data = data;
405 361
406 mutex_lock(&info->results_lock); 362static unsigned long long dmatest_persec(s64 runtime, unsigned int val)
407 list_add_tail(&tr->node, &r->results);
408 mutex_unlock(&info->results_lock);
409
410 if (tr->type == DMATEST_ET_OK)
411 pr_debug("%s\n", thread_result_get(r->name, tr));
412 else
413 pr_warn("%s\n", thread_result_get(r->name, tr));
414
415 return 0;
416}
417
418static unsigned int verify_result_add(struct dmatest_info *info,
419 struct dmatest_result *r, unsigned int n,
420 unsigned int src_off, unsigned int dst_off, unsigned int len,
421 u8 **bufs, int whence, unsigned int counter, u8 pattern,
422 bool is_srcbuf)
423{ 363{
424 struct dmatest_verify_result *vr; 364 unsigned long long per_sec = 1000000;
425 unsigned int error_count;
426 unsigned int buf_off = is_srcbuf ? src_off : dst_off;
427 unsigned int start, end;
428
429 if (whence < 0) {
430 start = 0;
431 end = buf_off;
432 } else if (whence > 0) {
433 start = buf_off + len;
434 end = info->params.buf_size;
435 } else {
436 start = buf_off;
437 end = buf_off + len;
438 }
439 365
440 vr = kmalloc(sizeof(*vr), GFP_KERNEL); 366 if (runtime <= 0)
441 if (!vr) { 367 return 0;
442 pr_warn("dmatest: No memory to store verify result\n");
443 return dmatest_verify(NULL, bufs, start, end, counter, pattern,
444 is_srcbuf);
445 }
446
447 vr->pattern = pattern;
448 vr->is_srcbuf = is_srcbuf;
449
450 error_count = dmatest_verify(vr, bufs, start, end, counter, pattern,
451 is_srcbuf);
452 if (error_count) {
453 vr->error_count = error_count;
454 thread_result_add(info, r, DMATEST_ET_VERIFY_BUF, n, src_off,
455 dst_off, len, (unsigned long)vr);
456 return error_count;
457 }
458
459 kfree(vr);
460 return 0;
461}
462
463static void result_free(struct dmatest_info *info, const char *name)
464{
465 struct dmatest_result *r, *_r;
466
467 mutex_lock(&info->results_lock);
468 list_for_each_entry_safe(r, _r, &info->results, node) {
469 struct dmatest_thread_result *tr, *_tr;
470
471 if (name && strcmp(r->name, name))
472 continue;
473
474 list_for_each_entry_safe(tr, _tr, &r->results, node) {
475 if (tr->type == DMATEST_ET_VERIFY_BUF)
476 kfree(tr->vr);
477 list_del(&tr->node);
478 kfree(tr);
479 }
480 368
481 kfree(r->name); 369 /* drop precision until runtime is 32-bits */
482 list_del(&r->node); 370 while (runtime > UINT_MAX) {
483 kfree(r); 371 runtime >>= 1;
372 per_sec <<= 1;
484 } 373 }
485 374
486 mutex_unlock(&info->results_lock); 375 per_sec *= val;
376 do_div(per_sec, runtime);
377 return per_sec;
487} 378}
488 379
489static struct dmatest_result *result_init(struct dmatest_info *info, 380static unsigned long long dmatest_KBs(s64 runtime, unsigned long long len)
490 const char *name)
491{ 381{
492 struct dmatest_result *r; 382 return dmatest_persec(runtime, len >> 10);
493
494 r = kzalloc(sizeof(*r), GFP_KERNEL);
495 if (r) {
496 r->name = kstrdup(name, GFP_KERNEL);
497 INIT_LIST_HEAD(&r->results);
498 mutex_lock(&info->results_lock);
499 list_add_tail(&r->node, &info->results);
500 mutex_unlock(&info->results_lock);
501 }
502 return r;
503} 383}
504 384
505/* 385/*
@@ -525,7 +405,6 @@ static int dmatest_func(void *data)
525 struct dmatest_params *params; 405 struct dmatest_params *params;
526 struct dma_chan *chan; 406 struct dma_chan *chan;
527 struct dma_device *dev; 407 struct dma_device *dev;
528 const char *thread_name;
529 unsigned int src_off, dst_off, len; 408 unsigned int src_off, dst_off, len;
530 unsigned int error_count; 409 unsigned int error_count;
531 unsigned int failed_tests = 0; 410 unsigned int failed_tests = 0;
@@ -538,9 +417,10 @@ static int dmatest_func(void *data)
538 int src_cnt; 417 int src_cnt;
539 int dst_cnt; 418 int dst_cnt;
540 int i; 419 int i;
541 struct dmatest_result *result; 420 ktime_t ktime;
421 s64 runtime = 0;
422 unsigned long long total_len = 0;
542 423
543 thread_name = current->comm;
544 set_freezable(); 424 set_freezable();
545 425
546 ret = -ENOMEM; 426 ret = -ENOMEM;
@@ -570,10 +450,6 @@ static int dmatest_func(void *data)
570 } else 450 } else
571 goto err_thread_type; 451 goto err_thread_type;
572 452
573 result = result_init(info, thread_name);
574 if (!result)
575 goto err_srcs;
576
577 thread->srcs = kcalloc(src_cnt+1, sizeof(u8 *), GFP_KERNEL); 453 thread->srcs = kcalloc(src_cnt+1, sizeof(u8 *), GFP_KERNEL);
578 if (!thread->srcs) 454 if (!thread->srcs)
579 goto err_srcs; 455 goto err_srcs;
@@ -597,17 +473,17 @@ static int dmatest_func(void *data)
597 set_user_nice(current, 10); 473 set_user_nice(current, 10);
598 474
599 /* 475 /*
600 * src buffers are freed by the DMAEngine code with dma_unmap_single() 476 * src and dst buffers are freed by ourselves below
601 * dst buffers are freed by ourselves below
602 */ 477 */
603 flags = DMA_CTRL_ACK | DMA_PREP_INTERRUPT 478 flags = DMA_CTRL_ACK | DMA_PREP_INTERRUPT;
604 | DMA_COMPL_SKIP_DEST_UNMAP | DMA_COMPL_SRC_UNMAP_SINGLE;
605 479
480 ktime = ktime_get();
606 while (!kthread_should_stop() 481 while (!kthread_should_stop()
607 && !(params->iterations && total_tests >= params->iterations)) { 482 && !(params->iterations && total_tests >= params->iterations)) {
608 struct dma_async_tx_descriptor *tx = NULL; 483 struct dma_async_tx_descriptor *tx = NULL;
609 dma_addr_t dma_srcs[src_cnt]; 484 struct dmaengine_unmap_data *um;
610 dma_addr_t dma_dsts[dst_cnt]; 485 dma_addr_t srcs[src_cnt];
486 dma_addr_t *dsts;
611 u8 align = 0; 487 u8 align = 0;
612 488
613 total_tests++; 489 total_tests++;
@@ -626,81 +502,103 @@ static int dmatest_func(void *data)
626 break; 502 break;
627 } 503 }
628 504
629 len = dmatest_random() % params->buf_size + 1; 505 if (params->noverify) {
506 len = params->buf_size;
507 src_off = 0;
508 dst_off = 0;
509 } else {
510 len = dmatest_random() % params->buf_size + 1;
511 len = (len >> align) << align;
512 if (!len)
513 len = 1 << align;
514 src_off = dmatest_random() % (params->buf_size - len + 1);
515 dst_off = dmatest_random() % (params->buf_size - len + 1);
516
517 src_off = (src_off >> align) << align;
518 dst_off = (dst_off >> align) << align;
519
520 dmatest_init_srcs(thread->srcs, src_off, len,
521 params->buf_size);
522 dmatest_init_dsts(thread->dsts, dst_off, len,
523 params->buf_size);
524 }
525
630 len = (len >> align) << align; 526 len = (len >> align) << align;
631 if (!len) 527 if (!len)
632 len = 1 << align; 528 len = 1 << align;
633 src_off = dmatest_random() % (params->buf_size - len + 1); 529 total_len += len;
634 dst_off = dmatest_random() % (params->buf_size - len + 1);
635 530
636 src_off = (src_off >> align) << align; 531 um = dmaengine_get_unmap_data(dev->dev, src_cnt+dst_cnt,
637 dst_off = (dst_off >> align) << align; 532 GFP_KERNEL);
638 533 if (!um) {
639 dmatest_init_srcs(thread->srcs, src_off, len, params->buf_size); 534 failed_tests++;
640 dmatest_init_dsts(thread->dsts, dst_off, len, params->buf_size); 535 result("unmap data NULL", total_tests,
536 src_off, dst_off, len, ret);
537 continue;
538 }
641 539
540 um->len = params->buf_size;
642 for (i = 0; i < src_cnt; i++) { 541 for (i = 0; i < src_cnt; i++) {
643 u8 *buf = thread->srcs[i] + src_off; 542 unsigned long buf = (unsigned long) thread->srcs[i];
644 543 struct page *pg = virt_to_page(buf);
645 dma_srcs[i] = dma_map_single(dev->dev, buf, len, 544 unsigned pg_off = buf & ~PAGE_MASK;
646 DMA_TO_DEVICE); 545
647 ret = dma_mapping_error(dev->dev, dma_srcs[i]); 546 um->addr[i] = dma_map_page(dev->dev, pg, pg_off,
547 um->len, DMA_TO_DEVICE);
548 srcs[i] = um->addr[i] + src_off;
549 ret = dma_mapping_error(dev->dev, um->addr[i]);
648 if (ret) { 550 if (ret) {
649 unmap_src(dev->dev, dma_srcs, len, i); 551 dmaengine_unmap_put(um);
650 thread_result_add(info, result, 552 result("src mapping error", total_tests,
651 DMATEST_ET_MAP_SRC, 553 src_off, dst_off, len, ret);
652 total_tests, src_off, dst_off,
653 len, ret);
654 failed_tests++; 554 failed_tests++;
655 continue; 555 continue;
656 } 556 }
557 um->to_cnt++;
657 } 558 }
658 /* map with DMA_BIDIRECTIONAL to force writeback/invalidate */ 559 /* map with DMA_BIDIRECTIONAL to force writeback/invalidate */
560 dsts = &um->addr[src_cnt];
659 for (i = 0; i < dst_cnt; i++) { 561 for (i = 0; i < dst_cnt; i++) {
660 dma_dsts[i] = dma_map_single(dev->dev, thread->dsts[i], 562 unsigned long buf = (unsigned long) thread->dsts[i];
661 params->buf_size, 563 struct page *pg = virt_to_page(buf);
662 DMA_BIDIRECTIONAL); 564 unsigned pg_off = buf & ~PAGE_MASK;
663 ret = dma_mapping_error(dev->dev, dma_dsts[i]); 565
566 dsts[i] = dma_map_page(dev->dev, pg, pg_off, um->len,
567 DMA_BIDIRECTIONAL);
568 ret = dma_mapping_error(dev->dev, dsts[i]);
664 if (ret) { 569 if (ret) {
665 unmap_src(dev->dev, dma_srcs, len, src_cnt); 570 dmaengine_unmap_put(um);
666 unmap_dst(dev->dev, dma_dsts, params->buf_size, 571 result("dst mapping error", total_tests,
667 i); 572 src_off, dst_off, len, ret);
668 thread_result_add(info, result,
669 DMATEST_ET_MAP_DST,
670 total_tests, src_off, dst_off,
671 len, ret);
672 failed_tests++; 573 failed_tests++;
673 continue; 574 continue;
674 } 575 }
576 um->bidi_cnt++;
675 } 577 }
676 578
677 if (thread->type == DMA_MEMCPY) 579 if (thread->type == DMA_MEMCPY)
678 tx = dev->device_prep_dma_memcpy(chan, 580 tx = dev->device_prep_dma_memcpy(chan,
679 dma_dsts[0] + dst_off, 581 dsts[0] + dst_off,
680 dma_srcs[0], len, 582 srcs[0], len, flags);
681 flags);
682 else if (thread->type == DMA_XOR) 583 else if (thread->type == DMA_XOR)
683 tx = dev->device_prep_dma_xor(chan, 584 tx = dev->device_prep_dma_xor(chan,
684 dma_dsts[0] + dst_off, 585 dsts[0] + dst_off,
685 dma_srcs, src_cnt, 586 srcs, src_cnt,
686 len, flags); 587 len, flags);
687 else if (thread->type == DMA_PQ) { 588 else if (thread->type == DMA_PQ) {
688 dma_addr_t dma_pq[dst_cnt]; 589 dma_addr_t dma_pq[dst_cnt];
689 590
690 for (i = 0; i < dst_cnt; i++) 591 for (i = 0; i < dst_cnt; i++)
691 dma_pq[i] = dma_dsts[i] + dst_off; 592 dma_pq[i] = dsts[i] + dst_off;
692 tx = dev->device_prep_dma_pq(chan, dma_pq, dma_srcs, 593 tx = dev->device_prep_dma_pq(chan, dma_pq, srcs,
693 src_cnt, pq_coefs, 594 src_cnt, pq_coefs,
694 len, flags); 595 len, flags);
695 } 596 }
696 597
697 if (!tx) { 598 if (!tx) {
698 unmap_src(dev->dev, dma_srcs, len, src_cnt); 599 dmaengine_unmap_put(um);
699 unmap_dst(dev->dev, dma_dsts, params->buf_size, 600 result("prep error", total_tests, src_off,
700 dst_cnt); 601 dst_off, len, ret);
701 thread_result_add(info, result, DMATEST_ET_PREP,
702 total_tests, src_off, dst_off,
703 len, 0);
704 msleep(100); 602 msleep(100);
705 failed_tests++; 603 failed_tests++;
706 continue; 604 continue;
@@ -712,9 +610,9 @@ static int dmatest_func(void *data)
712 cookie = tx->tx_submit(tx); 610 cookie = tx->tx_submit(tx);
713 611
714 if (dma_submit_error(cookie)) { 612 if (dma_submit_error(cookie)) {
715 thread_result_add(info, result, DMATEST_ET_SUBMIT, 613 dmaengine_unmap_put(um);
716 total_tests, src_off, dst_off, 614 result("submit error", total_tests, src_off,
717 len, cookie); 615 dst_off, len, ret);
718 msleep(100); 616 msleep(100);
719 failed_tests++; 617 failed_tests++;
720 continue; 618 continue;
@@ -735,59 +633,59 @@ static int dmatest_func(void *data)
735 * free it this time?" dancing. For now, just 633 * free it this time?" dancing. For now, just
736 * leave it dangling. 634 * leave it dangling.
737 */ 635 */
738 thread_result_add(info, result, DMATEST_ET_TIMEOUT, 636 dmaengine_unmap_put(um);
739 total_tests, src_off, dst_off, 637 result("test timed out", total_tests, src_off, dst_off,
740 len, 0); 638 len, 0);
741 failed_tests++; 639 failed_tests++;
742 continue; 640 continue;
743 } else if (status != DMA_COMPLETE) { 641 } else if (status != DMA_COMPLETE) {
744 enum dmatest_error_type type = (status == DMA_ERROR) ? 642 dmaengine_unmap_put(um);
745 DMATEST_ET_DMA_ERROR : DMATEST_ET_DMA_IN_PROGRESS; 643 result(status == DMA_ERROR ?
746 thread_result_add(info, result, type, 644 "completion error status" :
747 total_tests, src_off, dst_off, 645 "completion busy status", total_tests, src_off,
748 len, status); 646 dst_off, len, ret);
749 failed_tests++; 647 failed_tests++;
750 continue; 648 continue;
751 } 649 }
752 650
753 /* Unmap by myself (see DMA_COMPL_SKIP_DEST_UNMAP above) */ 651 dmaengine_unmap_put(um);
754 unmap_dst(dev->dev, dma_dsts, params->buf_size, dst_cnt);
755 652
756 error_count = 0; 653 if (params->noverify) {
654 verbose_result("test passed", total_tests, src_off,
655 dst_off, len, 0);
656 continue;
657 }
757 658
758 pr_debug("%s: verifying source buffer...\n", thread_name); 659 pr_debug("%s: verifying source buffer...\n", current->comm);
759 error_count += verify_result_add(info, result, total_tests, 660 error_count = dmatest_verify(thread->srcs, 0, src_off,
760 src_off, dst_off, len, thread->srcs, -1,
761 0, PATTERN_SRC, true); 661 0, PATTERN_SRC, true);
762 error_count += verify_result_add(info, result, total_tests, 662 error_count += dmatest_verify(thread->srcs, src_off,
763 src_off, dst_off, len, thread->srcs, 0, 663 src_off + len, src_off,
764 src_off, PATTERN_SRC | PATTERN_COPY, true); 664 PATTERN_SRC | PATTERN_COPY, true);
765 error_count += verify_result_add(info, result, total_tests, 665 error_count += dmatest_verify(thread->srcs, src_off + len,
766 src_off, dst_off, len, thread->srcs, 1, 666 params->buf_size, src_off + len,
767 src_off + len, PATTERN_SRC, true); 667 PATTERN_SRC, true);
768 668
769 pr_debug("%s: verifying dest buffer...\n", thread_name); 669 pr_debug("%s: verifying dest buffer...\n", current->comm);
770 error_count += verify_result_add(info, result, total_tests, 670 error_count += dmatest_verify(thread->dsts, 0, dst_off,
771 src_off, dst_off, len, thread->dsts, -1,
772 0, PATTERN_DST, false); 671 0, PATTERN_DST, false);
773 error_count += verify_result_add(info, result, total_tests, 672 error_count += dmatest_verify(thread->dsts, dst_off,
774 src_off, dst_off, len, thread->dsts, 0, 673 dst_off + len, src_off,
775 src_off, PATTERN_SRC | PATTERN_COPY, false); 674 PATTERN_SRC | PATTERN_COPY, false);
776 error_count += verify_result_add(info, result, total_tests, 675 error_count += dmatest_verify(thread->dsts, dst_off + len,
777 src_off, dst_off, len, thread->dsts, 1, 676 params->buf_size, dst_off + len,
778 dst_off + len, PATTERN_DST, false); 677 PATTERN_DST, false);
779 678
780 if (error_count) { 679 if (error_count) {
781 thread_result_add(info, result, DMATEST_ET_VERIFY, 680 result("data error", total_tests, src_off, dst_off,
782 total_tests, src_off, dst_off, 681 len, error_count);
783 len, error_count);
784 failed_tests++; 682 failed_tests++;
785 } else { 683 } else {
786 thread_result_add(info, result, DMATEST_ET_OK, 684 verbose_result("test passed", total_tests, src_off,
787 total_tests, src_off, dst_off, 685 dst_off, len, 0);
788 len, 0);
789 } 686 }
790 } 687 }
688 runtime = ktime_us_delta(ktime_get(), ktime);
791 689
792 ret = 0; 690 ret = 0;
793 for (i = 0; thread->dsts[i]; i++) 691 for (i = 0; thread->dsts[i]; i++)
@@ -802,20 +700,17 @@ err_srcbuf:
802err_srcs: 700err_srcs:
803 kfree(pq_coefs); 701 kfree(pq_coefs);
804err_thread_type: 702err_thread_type:
805 pr_notice("%s: terminating after %u tests, %u failures (status %d)\n", 703 pr_info("%s: summary %u tests, %u failures %llu iops %llu KB/s (%d)\n",
806 thread_name, total_tests, failed_tests, ret); 704 current->comm, total_tests, failed_tests,
705 dmatest_persec(runtime, total_tests),
706 dmatest_KBs(runtime, total_len), ret);
807 707
808 /* terminate all transfers on specified channels */ 708 /* terminate all transfers on specified channels */
809 if (ret) 709 if (ret)
810 dmaengine_terminate_all(chan); 710 dmaengine_terminate_all(chan);
811 711
812 thread->done = true; 712 thread->done = true;
813 713 wake_up(&thread_wait);
814 if (params->iterations > 0)
815 while (!kthread_should_stop()) {
816 DECLARE_WAIT_QUEUE_HEAD_ONSTACK(wait_dmatest_exit);
817 interruptible_sleep_on(&wait_dmatest_exit);
818 }
819 714
820 return ret; 715 return ret;
821} 716}
@@ -828,9 +723,10 @@ static void dmatest_cleanup_channel(struct dmatest_chan *dtc)
828 723
829 list_for_each_entry_safe(thread, _thread, &dtc->threads, node) { 724 list_for_each_entry_safe(thread, _thread, &dtc->threads, node) {
830 ret = kthread_stop(thread->task); 725 ret = kthread_stop(thread->task);
831 pr_debug("dmatest: thread %s exited with status %d\n", 726 pr_debug("thread %s exited with status %d\n",
832 thread->task->comm, ret); 727 thread->task->comm, ret);
833 list_del(&thread->node); 728 list_del(&thread->node);
729 put_task_struct(thread->task);
834 kfree(thread); 730 kfree(thread);
835 } 731 }
836 732
@@ -861,27 +757,27 @@ static int dmatest_add_threads(struct dmatest_info *info,
861 for (i = 0; i < params->threads_per_chan; i++) { 757 for (i = 0; i < params->threads_per_chan; i++) {
862 thread = kzalloc(sizeof(struct dmatest_thread), GFP_KERNEL); 758 thread = kzalloc(sizeof(struct dmatest_thread), GFP_KERNEL);
863 if (!thread) { 759 if (!thread) {
864 pr_warning("dmatest: No memory for %s-%s%u\n", 760 pr_warn("No memory for %s-%s%u\n",
865 dma_chan_name(chan), op, i); 761 dma_chan_name(chan), op, i);
866
867 break; 762 break;
868 } 763 }
869 thread->info = info; 764 thread->info = info;
870 thread->chan = dtc->chan; 765 thread->chan = dtc->chan;
871 thread->type = type; 766 thread->type = type;
872 smp_wmb(); 767 smp_wmb();
873 thread->task = kthread_run(dmatest_func, thread, "%s-%s%u", 768 thread->task = kthread_create(dmatest_func, thread, "%s-%s%u",
874 dma_chan_name(chan), op, i); 769 dma_chan_name(chan), op, i);
875 if (IS_ERR(thread->task)) { 770 if (IS_ERR(thread->task)) {
876 pr_warning("dmatest: Failed to run thread %s-%s%u\n", 771 pr_warn("Failed to create thread %s-%s%u\n",
877 dma_chan_name(chan), op, i); 772 dma_chan_name(chan), op, i);
878 kfree(thread); 773 kfree(thread);
879 break; 774 break;
880 } 775 }
881 776
882 /* srcbuf and dstbuf are allocated by the thread itself */ 777 /* srcbuf and dstbuf are allocated by the thread itself */
883 778 get_task_struct(thread->task);
884 list_add_tail(&thread->node, &dtc->threads); 779 list_add_tail(&thread->node, &dtc->threads);
780 wake_up_process(thread->task);
885 } 781 }
886 782
887 return i; 783 return i;
@@ -897,7 +793,7 @@ static int dmatest_add_channel(struct dmatest_info *info,
897 793
898 dtc = kmalloc(sizeof(struct dmatest_chan), GFP_KERNEL); 794 dtc = kmalloc(sizeof(struct dmatest_chan), GFP_KERNEL);
899 if (!dtc) { 795 if (!dtc) {
900 pr_warning("dmatest: No memory for %s\n", dma_chan_name(chan)); 796 pr_warn("No memory for %s\n", dma_chan_name(chan));
901 return -ENOMEM; 797 return -ENOMEM;
902 } 798 }
903 799
@@ -917,7 +813,7 @@ static int dmatest_add_channel(struct dmatest_info *info,
917 thread_count += cnt > 0 ? cnt : 0; 813 thread_count += cnt > 0 ? cnt : 0;
918 } 814 }
919 815
920 pr_info("dmatest: Started %u threads using %s\n", 816 pr_info("Started %u threads using %s\n",
921 thread_count, dma_chan_name(chan)); 817 thread_count, dma_chan_name(chan));
922 818
923 list_add_tail(&dtc->node, &info->channels); 819 list_add_tail(&dtc->node, &info->channels);
@@ -937,20 +833,20 @@ static bool filter(struct dma_chan *chan, void *param)
937 return true; 833 return true;
938} 834}
939 835
940static int __run_threaded_test(struct dmatest_info *info) 836static void request_channels(struct dmatest_info *info,
837 enum dma_transaction_type type)
941{ 838{
942 dma_cap_mask_t mask; 839 dma_cap_mask_t mask;
943 struct dma_chan *chan;
944 struct dmatest_params *params = &info->params;
945 int err = 0;
946 840
947 dma_cap_zero(mask); 841 dma_cap_zero(mask);
948 dma_cap_set(DMA_MEMCPY, mask); 842 dma_cap_set(type, mask);
949 for (;;) { 843 for (;;) {
844 struct dmatest_params *params = &info->params;
845 struct dma_chan *chan;
846
950 chan = dma_request_channel(mask, filter, params); 847 chan = dma_request_channel(mask, filter, params);
951 if (chan) { 848 if (chan) {
952 err = dmatest_add_channel(info, chan); 849 if (dmatest_add_channel(info, chan)) {
953 if (err) {
954 dma_release_channel(chan); 850 dma_release_channel(chan);
955 break; /* add_channel failed, punt */ 851 break; /* add_channel failed, punt */
956 } 852 }
@@ -960,22 +856,30 @@ static int __run_threaded_test(struct dmatest_info *info)
960 info->nr_channels >= params->max_channels) 856 info->nr_channels >= params->max_channels)
961 break; /* we have all we need */ 857 break; /* we have all we need */
962 } 858 }
963 return err;
964} 859}
965 860
966#ifndef MODULE 861static void run_threaded_test(struct dmatest_info *info)
967static int run_threaded_test(struct dmatest_info *info)
968{ 862{
969 int ret; 863 struct dmatest_params *params = &info->params;
970 864
971 mutex_lock(&info->lock); 865 /* Copy test parameters */
972 ret = __run_threaded_test(info); 866 params->buf_size = test_buf_size;
973 mutex_unlock(&info->lock); 867 strlcpy(params->channel, strim(test_channel), sizeof(params->channel));
974 return ret; 868 strlcpy(params->device, strim(test_device), sizeof(params->device));
869 params->threads_per_chan = threads_per_chan;
870 params->max_channels = max_channels;
871 params->iterations = iterations;
872 params->xor_sources = xor_sources;
873 params->pq_sources = pq_sources;
874 params->timeout = timeout;
875 params->noverify = noverify;
876
877 request_channels(info, DMA_MEMCPY);
878 request_channels(info, DMA_XOR);
879 request_channels(info, DMA_PQ);
975} 880}
976#endif
977 881
978static void __stop_threaded_test(struct dmatest_info *info) 882static void stop_threaded_test(struct dmatest_info *info)
979{ 883{
980 struct dmatest_chan *dtc, *_dtc; 884 struct dmatest_chan *dtc, *_dtc;
981 struct dma_chan *chan; 885 struct dma_chan *chan;
@@ -984,203 +888,86 @@ static void __stop_threaded_test(struct dmatest_info *info)
984 list_del(&dtc->node); 888 list_del(&dtc->node);
985 chan = dtc->chan; 889 chan = dtc->chan;
986 dmatest_cleanup_channel(dtc); 890 dmatest_cleanup_channel(dtc);
987 pr_debug("dmatest: dropped channel %s\n", dma_chan_name(chan)); 891 pr_debug("dropped channel %s\n", dma_chan_name(chan));
988 dma_release_channel(chan); 892 dma_release_channel(chan);
989 } 893 }
990 894
991 info->nr_channels = 0; 895 info->nr_channels = 0;
992} 896}
993 897
994static void stop_threaded_test(struct dmatest_info *info) 898static void restart_threaded_test(struct dmatest_info *info, bool run)
995{ 899{
996 mutex_lock(&info->lock); 900 /* we might be called early to set run=, defer running until all
997 __stop_threaded_test(info); 901 * parameters have been evaluated
998 mutex_unlock(&info->lock); 902 */
999} 903 if (!info->did_init)
1000 904 return;
1001static int __restart_threaded_test(struct dmatest_info *info, bool run)
1002{
1003 struct dmatest_params *params = &info->params;
1004 905
1005 /* Stop any running test first */ 906 /* Stop any running test first */
1006 __stop_threaded_test(info); 907 stop_threaded_test(info);
1007
1008 if (run == false)
1009 return 0;
1010
1011 /* Clear results from previous run */
1012 result_free(info, NULL);
1013
1014 /* Copy test parameters */
1015 params->buf_size = test_buf_size;
1016 strlcpy(params->channel, strim(test_channel), sizeof(params->channel));
1017 strlcpy(params->device, strim(test_device), sizeof(params->device));
1018 params->threads_per_chan = threads_per_chan;
1019 params->max_channels = max_channels;
1020 params->iterations = iterations;
1021 params->xor_sources = xor_sources;
1022 params->pq_sources = pq_sources;
1023 params->timeout = timeout;
1024 908
1025 /* Run test with new parameters */ 909 /* Run test with new parameters */
1026 return __run_threaded_test(info); 910 run_threaded_test(info);
1027}
1028
1029static bool __is_threaded_test_run(struct dmatest_info *info)
1030{
1031 struct dmatest_chan *dtc;
1032
1033 list_for_each_entry(dtc, &info->channels, node) {
1034 struct dmatest_thread *thread;
1035
1036 list_for_each_entry(thread, &dtc->threads, node) {
1037 if (!thread->done)
1038 return true;
1039 }
1040 }
1041
1042 return false;
1043} 911}
1044 912
1045static ssize_t dtf_read_run(struct file *file, char __user *user_buf, 913static int dmatest_run_get(char *val, const struct kernel_param *kp)
1046 size_t count, loff_t *ppos)
1047{ 914{
1048 struct dmatest_info *info = file->private_data; 915 struct dmatest_info *info = &test_info;
1049 char buf[3];
1050 916
1051 mutex_lock(&info->lock); 917 mutex_lock(&info->lock);
1052 918 if (is_threaded_test_run(info)) {
1053 if (__is_threaded_test_run(info)) { 919 dmatest_run = true;
1054 buf[0] = 'Y';
1055 } else { 920 } else {
1056 __stop_threaded_test(info); 921 stop_threaded_test(info);
1057 buf[0] = 'N'; 922 dmatest_run = false;
1058 } 923 }
1059
1060 mutex_unlock(&info->lock); 924 mutex_unlock(&info->lock);
1061 buf[1] = '\n';
1062 buf[2] = 0x00;
1063 return simple_read_from_buffer(user_buf, count, ppos, buf, 2);
1064}
1065
1066static ssize_t dtf_write_run(struct file *file, const char __user *user_buf,
1067 size_t count, loff_t *ppos)
1068{
1069 struct dmatest_info *info = file->private_data;
1070 char buf[16];
1071 bool bv;
1072 int ret = 0;
1073 925
1074 if (copy_from_user(buf, user_buf, min(count, (sizeof(buf) - 1)))) 926 return param_get_bool(val, kp);
1075 return -EFAULT;
1076
1077 if (strtobool(buf, &bv) == 0) {
1078 mutex_lock(&info->lock);
1079
1080 if (__is_threaded_test_run(info))
1081 ret = -EBUSY;
1082 else
1083 ret = __restart_threaded_test(info, bv);
1084
1085 mutex_unlock(&info->lock);
1086 }
1087
1088 return ret ? ret : count;
1089} 927}
1090 928
1091static const struct file_operations dtf_run_fops = { 929static int dmatest_run_set(const char *val, const struct kernel_param *kp)
1092 .read = dtf_read_run,
1093 .write = dtf_write_run,
1094 .open = simple_open,
1095 .llseek = default_llseek,
1096};
1097
1098static int dtf_results_show(struct seq_file *sf, void *data)
1099{ 930{
1100 struct dmatest_info *info = sf->private; 931 struct dmatest_info *info = &test_info;
1101 struct dmatest_result *result; 932 int ret;
1102 struct dmatest_thread_result *tr;
1103 unsigned int i;
1104 933
1105 mutex_lock(&info->results_lock); 934 mutex_lock(&info->lock);
1106 list_for_each_entry(result, &info->results, node) { 935 ret = param_set_bool(val, kp);
1107 list_for_each_entry(tr, &result->results, node) { 936 if (ret) {
1108 seq_printf(sf, "%s\n", 937 mutex_unlock(&info->lock);
1109 thread_result_get(result->name, tr)); 938 return ret;
1110 if (tr->type == DMATEST_ET_VERIFY_BUF) {
1111 for (i = 0; i < tr->vr->error_count; i++) {
1112 seq_printf(sf, "\t%s\n",
1113 verify_result_get_one(tr->vr, i));
1114 }
1115 }
1116 }
1117 } 939 }
1118 940
1119 mutex_unlock(&info->results_lock); 941 if (is_threaded_test_run(info))
1120 return 0; 942 ret = -EBUSY;
1121} 943 else if (dmatest_run)
1122 944 restart_threaded_test(info, dmatest_run);
1123static int dtf_results_open(struct inode *inode, struct file *file)
1124{
1125 return single_open(file, dtf_results_show, inode->i_private);
1126}
1127
1128static const struct file_operations dtf_results_fops = {
1129 .open = dtf_results_open,
1130 .read = seq_read,
1131 .llseek = seq_lseek,
1132 .release = single_release,
1133};
1134
1135static int dmatest_register_dbgfs(struct dmatest_info *info)
1136{
1137 struct dentry *d;
1138
1139 d = debugfs_create_dir("dmatest", NULL);
1140 if (IS_ERR(d))
1141 return PTR_ERR(d);
1142 if (!d)
1143 goto err_root;
1144 945
1145 info->root = d; 946 mutex_unlock(&info->lock);
1146
1147 /* Run or stop threaded test */
1148 debugfs_create_file("run", S_IWUSR | S_IRUGO, info->root, info,
1149 &dtf_run_fops);
1150
1151 /* Results of test in progress */
1152 debugfs_create_file("results", S_IRUGO, info->root, info,
1153 &dtf_results_fops);
1154
1155 return 0;
1156 947
1157err_root: 948 return ret;
1158 pr_err("dmatest: Failed to initialize debugfs\n");
1159 return -ENOMEM;
1160} 949}
1161 950
1162static int __init dmatest_init(void) 951static int __init dmatest_init(void)
1163{ 952{
1164 struct dmatest_info *info = &test_info; 953 struct dmatest_info *info = &test_info;
1165 int ret; 954 struct dmatest_params *params = &info->params;
1166
1167 memset(info, 0, sizeof(*info));
1168 955
1169 mutex_init(&info->lock); 956 if (dmatest_run) {
1170 INIT_LIST_HEAD(&info->channels); 957 mutex_lock(&info->lock);
958 run_threaded_test(info);
959 mutex_unlock(&info->lock);
960 }
1171 961
1172 mutex_init(&info->results_lock); 962 if (params->iterations && wait)
1173 INIT_LIST_HEAD(&info->results); 963 wait_event(thread_wait, !is_threaded_test_run(info));
1174 964
1175 ret = dmatest_register_dbgfs(info); 965 /* module parameters are stable, inittime tests are started,
1176 if (ret) 966 * let userspace take over 'run' control
1177 return ret; 967 */
968 info->did_init = true;
1178 969
1179#ifdef MODULE
1180 return 0; 970 return 0;
1181#else
1182 return run_threaded_test(info);
1183#endif
1184} 971}
1185/* when compiled-in wait for drivers to load first */ 972/* when compiled-in wait for drivers to load first */
1186late_initcall(dmatest_init); 973late_initcall(dmatest_init);
@@ -1189,9 +976,9 @@ static void __exit dmatest_exit(void)
1189{ 976{
1190 struct dmatest_info *info = &test_info; 977 struct dmatest_info *info = &test_info;
1191 978
1192 debugfs_remove_recursive(info->root); 979 mutex_lock(&info->lock);
1193 stop_threaded_test(info); 980 stop_threaded_test(info);
1194 result_free(info, NULL); 981 mutex_unlock(&info->lock);
1195} 982}
1196module_exit(dmatest_exit); 983module_exit(dmatest_exit);
1197 984
diff --git a/drivers/dma/dw/core.c b/drivers/dma/dw/core.c
index 2c29331571e4..7516be4677cf 100644
--- a/drivers/dma/dw/core.c
+++ b/drivers/dma/dw/core.c
@@ -85,10 +85,6 @@ static struct device *chan2dev(struct dma_chan *chan)
85{ 85{
86 return &chan->dev->device; 86 return &chan->dev->device;
87} 87}
88static struct device *chan2parent(struct dma_chan *chan)
89{
90 return chan->dev->device.parent;
91}
92 88
93static struct dw_desc *dwc_first_active(struct dw_dma_chan *dwc) 89static struct dw_desc *dwc_first_active(struct dw_dma_chan *dwc)
94{ 90{
@@ -311,26 +307,7 @@ dwc_descriptor_complete(struct dw_dma_chan *dwc, struct dw_desc *desc,
311 list_splice_init(&desc->tx_list, &dwc->free_list); 307 list_splice_init(&desc->tx_list, &dwc->free_list);
312 list_move(&desc->desc_node, &dwc->free_list); 308 list_move(&desc->desc_node, &dwc->free_list);
313 309
314 if (!is_slave_direction(dwc->direction)) { 310 dma_descriptor_unmap(txd);
315 struct device *parent = chan2parent(&dwc->chan);
316 if (!(txd->flags & DMA_COMPL_SKIP_DEST_UNMAP)) {
317 if (txd->flags & DMA_COMPL_DEST_UNMAP_SINGLE)
318 dma_unmap_single(parent, desc->lli.dar,
319 desc->total_len, DMA_FROM_DEVICE);
320 else
321 dma_unmap_page(parent, desc->lli.dar,
322 desc->total_len, DMA_FROM_DEVICE);
323 }
324 if (!(txd->flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
325 if (txd->flags & DMA_COMPL_SRC_UNMAP_SINGLE)
326 dma_unmap_single(parent, desc->lli.sar,
327 desc->total_len, DMA_TO_DEVICE);
328 else
329 dma_unmap_page(parent, desc->lli.sar,
330 desc->total_len, DMA_TO_DEVICE);
331 }
332 }
333
334 spin_unlock_irqrestore(&dwc->lock, flags); 311 spin_unlock_irqrestore(&dwc->lock, flags);
335 312
336 if (callback) 313 if (callback)
diff --git a/drivers/dma/ep93xx_dma.c b/drivers/dma/ep93xx_dma.c
index 591cd8c63abb..cb4bf682a708 100644
--- a/drivers/dma/ep93xx_dma.c
+++ b/drivers/dma/ep93xx_dma.c
@@ -733,28 +733,6 @@ static void ep93xx_dma_advance_work(struct ep93xx_dma_chan *edmac)
733 spin_unlock_irqrestore(&edmac->lock, flags); 733 spin_unlock_irqrestore(&edmac->lock, flags);
734} 734}
735 735
736static void ep93xx_dma_unmap_buffers(struct ep93xx_dma_desc *desc)
737{
738 struct device *dev = desc->txd.chan->device->dev;
739
740 if (!(desc->txd.flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
741 if (desc->txd.flags & DMA_COMPL_SRC_UNMAP_SINGLE)
742 dma_unmap_single(dev, desc->src_addr, desc->size,
743 DMA_TO_DEVICE);
744 else
745 dma_unmap_page(dev, desc->src_addr, desc->size,
746 DMA_TO_DEVICE);
747 }
748 if (!(desc->txd.flags & DMA_COMPL_SKIP_DEST_UNMAP)) {
749 if (desc->txd.flags & DMA_COMPL_DEST_UNMAP_SINGLE)
750 dma_unmap_single(dev, desc->dst_addr, desc->size,
751 DMA_FROM_DEVICE);
752 else
753 dma_unmap_page(dev, desc->dst_addr, desc->size,
754 DMA_FROM_DEVICE);
755 }
756}
757
758static void ep93xx_dma_tasklet(unsigned long data) 736static void ep93xx_dma_tasklet(unsigned long data)
759{ 737{
760 struct ep93xx_dma_chan *edmac = (struct ep93xx_dma_chan *)data; 738 struct ep93xx_dma_chan *edmac = (struct ep93xx_dma_chan *)data;
@@ -787,13 +765,7 @@ static void ep93xx_dma_tasklet(unsigned long data)
787 765
788 /* Now we can release all the chained descriptors */ 766 /* Now we can release all the chained descriptors */
789 list_for_each_entry_safe(desc, d, &list, node) { 767 list_for_each_entry_safe(desc, d, &list, node) {
790 /* 768 dma_descriptor_unmap(&desc->txd);
791 * For the memcpy channels the API requires us to unmap the
792 * buffers unless requested otherwise.
793 */
794 if (!edmac->chan.private)
795 ep93xx_dma_unmap_buffers(desc);
796
797 ep93xx_dma_desc_put(edmac, desc); 769 ep93xx_dma_desc_put(edmac, desc);
798 } 770 }
799 771
diff --git a/drivers/dma/fsldma.c b/drivers/dma/fsldma.c
index 4e898121707b..bd7f888b7601 100644
--- a/drivers/dma/fsldma.c
+++ b/drivers/dma/fsldma.c
@@ -868,22 +868,7 @@ static void fsldma_cleanup_descriptor(struct fsldma_chan *chan,
868 /* Run any dependencies */ 868 /* Run any dependencies */
869 dma_run_dependencies(txd); 869 dma_run_dependencies(txd);
870 870
871 /* Unmap the dst buffer, if requested */ 871 dma_descriptor_unmap(txd);
872 if (!(txd->flags & DMA_COMPL_SKIP_DEST_UNMAP)) {
873 if (txd->flags & DMA_COMPL_DEST_UNMAP_SINGLE)
874 dma_unmap_single(dev, dst, len, DMA_FROM_DEVICE);
875 else
876 dma_unmap_page(dev, dst, len, DMA_FROM_DEVICE);
877 }
878
879 /* Unmap the src buffer, if requested */
880 if (!(txd->flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
881 if (txd->flags & DMA_COMPL_SRC_UNMAP_SINGLE)
882 dma_unmap_single(dev, src, len, DMA_TO_DEVICE);
883 else
884 dma_unmap_page(dev, src, len, DMA_TO_DEVICE);
885 }
886
887#ifdef FSL_DMA_LD_DEBUG 872#ifdef FSL_DMA_LD_DEBUG
888 chan_dbg(chan, "LD %p free\n", desc); 873 chan_dbg(chan, "LD %p free\n", desc);
889#endif 874#endif
diff --git a/drivers/dma/ioat/dma.c b/drivers/dma/ioat/dma.c
index a0f0fce5a84e..1a49c777607c 100644
--- a/drivers/dma/ioat/dma.c
+++ b/drivers/dma/ioat/dma.c
@@ -531,21 +531,6 @@ static void ioat1_cleanup_event(unsigned long data)
531 writew(IOAT_CHANCTRL_RUN, ioat->base.reg_base + IOAT_CHANCTRL_OFFSET); 531 writew(IOAT_CHANCTRL_RUN, ioat->base.reg_base + IOAT_CHANCTRL_OFFSET);
532} 532}
533 533
534void ioat_dma_unmap(struct ioat_chan_common *chan, enum dma_ctrl_flags flags,
535 size_t len, struct ioat_dma_descriptor *hw)
536{
537 struct pci_dev *pdev = chan->device->pdev;
538 size_t offset = len - hw->size;
539
540 if (!(flags & DMA_COMPL_SKIP_DEST_UNMAP))
541 ioat_unmap(pdev, hw->dst_addr - offset, len,
542 PCI_DMA_FROMDEVICE, flags, 1);
543
544 if (!(flags & DMA_COMPL_SKIP_SRC_UNMAP))
545 ioat_unmap(pdev, hw->src_addr - offset, len,
546 PCI_DMA_TODEVICE, flags, 0);
547}
548
549dma_addr_t ioat_get_current_completion(struct ioat_chan_common *chan) 534dma_addr_t ioat_get_current_completion(struct ioat_chan_common *chan)
550{ 535{
551 dma_addr_t phys_complete; 536 dma_addr_t phys_complete;
@@ -602,7 +587,7 @@ static void __cleanup(struct ioat_dma_chan *ioat, dma_addr_t phys_complete)
602 dump_desc_dbg(ioat, desc); 587 dump_desc_dbg(ioat, desc);
603 if (tx->cookie) { 588 if (tx->cookie) {
604 dma_cookie_complete(tx); 589 dma_cookie_complete(tx);
605 ioat_dma_unmap(chan, tx->flags, desc->len, desc->hw); 590 dma_descriptor_unmap(tx);
606 ioat->active -= desc->hw->tx_cnt; 591 ioat->active -= desc->hw->tx_cnt;
607 if (tx->callback) { 592 if (tx->callback) {
608 tx->callback(tx->callback_param); 593 tx->callback(tx->callback_param);
@@ -833,8 +818,7 @@ int ioat_dma_self_test(struct ioatdma_device *device)
833 818
834 dma_src = dma_map_single(dev, src, IOAT_TEST_SIZE, DMA_TO_DEVICE); 819 dma_src = dma_map_single(dev, src, IOAT_TEST_SIZE, DMA_TO_DEVICE);
835 dma_dest = dma_map_single(dev, dest, IOAT_TEST_SIZE, DMA_FROM_DEVICE); 820 dma_dest = dma_map_single(dev, dest, IOAT_TEST_SIZE, DMA_FROM_DEVICE);
836 flags = DMA_COMPL_SKIP_SRC_UNMAP | DMA_COMPL_SKIP_DEST_UNMAP | 821 flags = DMA_PREP_INTERRUPT;
837 DMA_PREP_INTERRUPT;
838 tx = device->common.device_prep_dma_memcpy(dma_chan, dma_dest, dma_src, 822 tx = device->common.device_prep_dma_memcpy(dma_chan, dma_dest, dma_src,
839 IOAT_TEST_SIZE, flags); 823 IOAT_TEST_SIZE, flags);
840 if (!tx) { 824 if (!tx) {
@@ -885,8 +869,7 @@ static char ioat_interrupt_style[32] = "msix";
885module_param_string(ioat_interrupt_style, ioat_interrupt_style, 869module_param_string(ioat_interrupt_style, ioat_interrupt_style,
886 sizeof(ioat_interrupt_style), 0644); 870 sizeof(ioat_interrupt_style), 0644);
887MODULE_PARM_DESC(ioat_interrupt_style, 871MODULE_PARM_DESC(ioat_interrupt_style,
888 "set ioat interrupt style: msix (default), " 872 "set ioat interrupt style: msix (default), msi, intx");
889 "msix-single-vector, msi, intx)");
890 873
891/** 874/**
892 * ioat_dma_setup_interrupts - setup interrupt handler 875 * ioat_dma_setup_interrupts - setup interrupt handler
@@ -904,8 +887,6 @@ int ioat_dma_setup_interrupts(struct ioatdma_device *device)
904 887
905 if (!strcmp(ioat_interrupt_style, "msix")) 888 if (!strcmp(ioat_interrupt_style, "msix"))
906 goto msix; 889 goto msix;
907 if (!strcmp(ioat_interrupt_style, "msix-single-vector"))
908 goto msix_single_vector;
909 if (!strcmp(ioat_interrupt_style, "msi")) 890 if (!strcmp(ioat_interrupt_style, "msi"))
910 goto msi; 891 goto msi;
911 if (!strcmp(ioat_interrupt_style, "intx")) 892 if (!strcmp(ioat_interrupt_style, "intx"))
@@ -920,10 +901,8 @@ msix:
920 device->msix_entries[i].entry = i; 901 device->msix_entries[i].entry = i;
921 902
922 err = pci_enable_msix(pdev, device->msix_entries, msixcnt); 903 err = pci_enable_msix(pdev, device->msix_entries, msixcnt);
923 if (err < 0) 904 if (err)
924 goto msi; 905 goto msi;
925 if (err > 0)
926 goto msix_single_vector;
927 906
928 for (i = 0; i < msixcnt; i++) { 907 for (i = 0; i < msixcnt; i++) {
929 msix = &device->msix_entries[i]; 908 msix = &device->msix_entries[i];
@@ -937,29 +916,13 @@ msix:
937 chan = ioat_chan_by_index(device, j); 916 chan = ioat_chan_by_index(device, j);
938 devm_free_irq(dev, msix->vector, chan); 917 devm_free_irq(dev, msix->vector, chan);
939 } 918 }
940 goto msix_single_vector; 919 goto msi;
941 } 920 }
942 } 921 }
943 intrctrl |= IOAT_INTRCTRL_MSIX_VECTOR_CONTROL; 922 intrctrl |= IOAT_INTRCTRL_MSIX_VECTOR_CONTROL;
944 device->irq_mode = IOAT_MSIX; 923 device->irq_mode = IOAT_MSIX;
945 goto done; 924 goto done;
946 925
947msix_single_vector:
948 msix = &device->msix_entries[0];
949 msix->entry = 0;
950 err = pci_enable_msix(pdev, device->msix_entries, 1);
951 if (err)
952 goto msi;
953
954 err = devm_request_irq(dev, msix->vector, ioat_dma_do_interrupt, 0,
955 "ioat-msix", device);
956 if (err) {
957 pci_disable_msix(pdev);
958 goto msi;
959 }
960 device->irq_mode = IOAT_MSIX_SINGLE;
961 goto done;
962
963msi: 926msi:
964 err = pci_enable_msi(pdev); 927 err = pci_enable_msi(pdev);
965 if (err) 928 if (err)
@@ -971,7 +934,7 @@ msi:
971 pci_disable_msi(pdev); 934 pci_disable_msi(pdev);
972 goto intx; 935 goto intx;
973 } 936 }
974 device->irq_mode = IOAT_MSIX; 937 device->irq_mode = IOAT_MSI;
975 goto done; 938 goto done;
976 939
977intx: 940intx:
diff --git a/drivers/dma/ioat/dma.h b/drivers/dma/ioat/dma.h
index 54fb7b9ff9aa..11fb877ddca9 100644
--- a/drivers/dma/ioat/dma.h
+++ b/drivers/dma/ioat/dma.h
@@ -52,7 +52,6 @@
52enum ioat_irq_mode { 52enum ioat_irq_mode {
53 IOAT_NOIRQ = 0, 53 IOAT_NOIRQ = 0,
54 IOAT_MSIX, 54 IOAT_MSIX,
55 IOAT_MSIX_SINGLE,
56 IOAT_MSI, 55 IOAT_MSI,
57 IOAT_INTX 56 IOAT_INTX
58}; 57};
@@ -83,7 +82,6 @@ struct ioatdma_device {
83 struct pci_pool *completion_pool; 82 struct pci_pool *completion_pool;
84#define MAX_SED_POOLS 5 83#define MAX_SED_POOLS 5
85 struct dma_pool *sed_hw_pool[MAX_SED_POOLS]; 84 struct dma_pool *sed_hw_pool[MAX_SED_POOLS];
86 struct kmem_cache *sed_pool;
87 struct dma_device common; 85 struct dma_device common;
88 u8 version; 86 u8 version;
89 struct msix_entry msix_entries[4]; 87 struct msix_entry msix_entries[4];
@@ -342,16 +340,6 @@ static inline bool is_ioat_bug(unsigned long err)
342 return !!err; 340 return !!err;
343} 341}
344 342
345static inline void ioat_unmap(struct pci_dev *pdev, dma_addr_t addr, size_t len,
346 int direction, enum dma_ctrl_flags flags, bool dst)
347{
348 if ((dst && (flags & DMA_COMPL_DEST_UNMAP_SINGLE)) ||
349 (!dst && (flags & DMA_COMPL_SRC_UNMAP_SINGLE)))
350 pci_unmap_single(pdev, addr, len, direction);
351 else
352 pci_unmap_page(pdev, addr, len, direction);
353}
354
355int ioat_probe(struct ioatdma_device *device); 343int ioat_probe(struct ioatdma_device *device);
356int ioat_register(struct ioatdma_device *device); 344int ioat_register(struct ioatdma_device *device);
357int ioat1_dma_probe(struct ioatdma_device *dev, int dca); 345int ioat1_dma_probe(struct ioatdma_device *dev, int dca);
@@ -363,8 +351,6 @@ void ioat_init_channel(struct ioatdma_device *device,
363 struct ioat_chan_common *chan, int idx); 351 struct ioat_chan_common *chan, int idx);
364enum dma_status ioat_dma_tx_status(struct dma_chan *c, dma_cookie_t cookie, 352enum dma_status ioat_dma_tx_status(struct dma_chan *c, dma_cookie_t cookie,
365 struct dma_tx_state *txstate); 353 struct dma_tx_state *txstate);
366void ioat_dma_unmap(struct ioat_chan_common *chan, enum dma_ctrl_flags flags,
367 size_t len, struct ioat_dma_descriptor *hw);
368bool ioat_cleanup_preamble(struct ioat_chan_common *chan, 354bool ioat_cleanup_preamble(struct ioat_chan_common *chan,
369 dma_addr_t *phys_complete); 355 dma_addr_t *phys_complete);
370void ioat_kobject_add(struct ioatdma_device *device, struct kobj_type *type); 356void ioat_kobject_add(struct ioatdma_device *device, struct kobj_type *type);
diff --git a/drivers/dma/ioat/dma_v2.c b/drivers/dma/ioat/dma_v2.c
index b925e1b1d139..5d3affe7e976 100644
--- a/drivers/dma/ioat/dma_v2.c
+++ b/drivers/dma/ioat/dma_v2.c
@@ -148,7 +148,7 @@ static void __cleanup(struct ioat2_dma_chan *ioat, dma_addr_t phys_complete)
148 tx = &desc->txd; 148 tx = &desc->txd;
149 dump_desc_dbg(ioat, desc); 149 dump_desc_dbg(ioat, desc);
150 if (tx->cookie) { 150 if (tx->cookie) {
151 ioat_dma_unmap(chan, tx->flags, desc->len, desc->hw); 151 dma_descriptor_unmap(tx);
152 dma_cookie_complete(tx); 152 dma_cookie_complete(tx);
153 if (tx->callback) { 153 if (tx->callback) {
154 tx->callback(tx->callback_param); 154 tx->callback(tx->callback_param);
diff --git a/drivers/dma/ioat/dma_v2.h b/drivers/dma/ioat/dma_v2.h
index 212d584fe427..470292767e68 100644
--- a/drivers/dma/ioat/dma_v2.h
+++ b/drivers/dma/ioat/dma_v2.h
@@ -157,7 +157,6 @@ static inline void ioat2_set_chainaddr(struct ioat2_dma_chan *ioat, u64 addr)
157 157
158int ioat2_dma_probe(struct ioatdma_device *dev, int dca); 158int ioat2_dma_probe(struct ioatdma_device *dev, int dca);
159int ioat3_dma_probe(struct ioatdma_device *dev, int dca); 159int ioat3_dma_probe(struct ioatdma_device *dev, int dca);
160void ioat3_dma_remove(struct ioatdma_device *dev);
161struct dca_provider *ioat2_dca_init(struct pci_dev *pdev, void __iomem *iobase); 160struct dca_provider *ioat2_dca_init(struct pci_dev *pdev, void __iomem *iobase);
162struct dca_provider *ioat3_dca_init(struct pci_dev *pdev, void __iomem *iobase); 161struct dca_provider *ioat3_dca_init(struct pci_dev *pdev, void __iomem *iobase);
163int ioat2_check_space_lock(struct ioat2_dma_chan *ioat, int num_descs); 162int ioat2_check_space_lock(struct ioat2_dma_chan *ioat, int num_descs);
diff --git a/drivers/dma/ioat/dma_v3.c b/drivers/dma/ioat/dma_v3.c
index 806b4ce5e38c..820817e97e62 100644
--- a/drivers/dma/ioat/dma_v3.c
+++ b/drivers/dma/ioat/dma_v3.c
@@ -67,6 +67,8 @@
67#include "dma.h" 67#include "dma.h"
68#include "dma_v2.h" 68#include "dma_v2.h"
69 69
70extern struct kmem_cache *ioat3_sed_cache;
71
70/* ioat hardware assumes at least two sources for raid operations */ 72/* ioat hardware assumes at least two sources for raid operations */
71#define src_cnt_to_sw(x) ((x) + 2) 73#define src_cnt_to_sw(x) ((x) + 2)
72#define src_cnt_to_hw(x) ((x) - 2) 74#define src_cnt_to_hw(x) ((x) - 2)
@@ -87,22 +89,8 @@ static const u8 pq_idx_to_field[] = { 1, 4, 5, 0, 1, 2, 4, 5 };
87static const u8 pq16_idx_to_field[] = { 1, 4, 1, 2, 3, 4, 5, 6, 7, 89static const u8 pq16_idx_to_field[] = { 1, 4, 1, 2, 3, 4, 5, 6, 7,
88 0, 1, 2, 3, 4, 5, 6 }; 90 0, 1, 2, 3, 4, 5, 6 };
89 91
90/*
91 * technically sources 1 and 2 do not require SED, but the op will have
92 * at least 9 descriptors so that's irrelevant.
93 */
94static const u8 pq16_idx_to_sed[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0,
95 1, 1, 1, 1, 1, 1, 1 };
96
97static void ioat3_eh(struct ioat2_dma_chan *ioat); 92static void ioat3_eh(struct ioat2_dma_chan *ioat);
98 93
99static dma_addr_t xor_get_src(struct ioat_raw_descriptor *descs[2], int idx)
100{
101 struct ioat_raw_descriptor *raw = descs[xor_idx_to_desc >> idx & 1];
102
103 return raw->field[xor_idx_to_field[idx]];
104}
105
106static void xor_set_src(struct ioat_raw_descriptor *descs[2], 94static void xor_set_src(struct ioat_raw_descriptor *descs[2],
107 dma_addr_t addr, u32 offset, int idx) 95 dma_addr_t addr, u32 offset, int idx)
108{ 96{
@@ -135,12 +123,6 @@ static void pq_set_src(struct ioat_raw_descriptor *descs[2],
135 pq->coef[idx] = coef; 123 pq->coef[idx] = coef;
136} 124}
137 125
138static int sed_get_pq16_pool_idx(int src_cnt)
139{
140
141 return pq16_idx_to_sed[src_cnt];
142}
143
144static bool is_jf_ioat(struct pci_dev *pdev) 126static bool is_jf_ioat(struct pci_dev *pdev)
145{ 127{
146 switch (pdev->device) { 128 switch (pdev->device) {
@@ -272,7 +254,7 @@ ioat3_alloc_sed(struct ioatdma_device *device, unsigned int hw_pool)
272 struct ioat_sed_ent *sed; 254 struct ioat_sed_ent *sed;
273 gfp_t flags = __GFP_ZERO | GFP_ATOMIC; 255 gfp_t flags = __GFP_ZERO | GFP_ATOMIC;
274 256
275 sed = kmem_cache_alloc(device->sed_pool, flags); 257 sed = kmem_cache_alloc(ioat3_sed_cache, flags);
276 if (!sed) 258 if (!sed)
277 return NULL; 259 return NULL;
278 260
@@ -280,7 +262,7 @@ ioat3_alloc_sed(struct ioatdma_device *device, unsigned int hw_pool)
280 sed->hw = dma_pool_alloc(device->sed_hw_pool[hw_pool], 262 sed->hw = dma_pool_alloc(device->sed_hw_pool[hw_pool],
281 flags, &sed->dma); 263 flags, &sed->dma);
282 if (!sed->hw) { 264 if (!sed->hw) {
283 kmem_cache_free(device->sed_pool, sed); 265 kmem_cache_free(ioat3_sed_cache, sed);
284 return NULL; 266 return NULL;
285 } 267 }
286 268
@@ -293,165 +275,7 @@ static void ioat3_free_sed(struct ioatdma_device *device, struct ioat_sed_ent *s
293 return; 275 return;
294 276
295 dma_pool_free(device->sed_hw_pool[sed->hw_pool], sed->hw, sed->dma); 277 dma_pool_free(device->sed_hw_pool[sed->hw_pool], sed->hw, sed->dma);
296 kmem_cache_free(device->sed_pool, sed); 278 kmem_cache_free(ioat3_sed_cache, sed);
297}
298
299static void ioat3_dma_unmap(struct ioat2_dma_chan *ioat,
300 struct ioat_ring_ent *desc, int idx)
301{
302 struct ioat_chan_common *chan = &ioat->base;
303 struct pci_dev *pdev = chan->device->pdev;
304 size_t len = desc->len;
305 size_t offset = len - desc->hw->size;
306 struct dma_async_tx_descriptor *tx = &desc->txd;
307 enum dma_ctrl_flags flags = tx->flags;
308
309 switch (desc->hw->ctl_f.op) {
310 case IOAT_OP_COPY:
311 if (!desc->hw->ctl_f.null) /* skip 'interrupt' ops */
312 ioat_dma_unmap(chan, flags, len, desc->hw);
313 break;
314 case IOAT_OP_XOR_VAL:
315 case IOAT_OP_XOR: {
316 struct ioat_xor_descriptor *xor = desc->xor;
317 struct ioat_ring_ent *ext;
318 struct ioat_xor_ext_descriptor *xor_ex = NULL;
319 int src_cnt = src_cnt_to_sw(xor->ctl_f.src_cnt);
320 struct ioat_raw_descriptor *descs[2];
321 int i;
322
323 if (src_cnt > 5) {
324 ext = ioat2_get_ring_ent(ioat, idx + 1);
325 xor_ex = ext->xor_ex;
326 }
327
328 if (!(flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
329 descs[0] = (struct ioat_raw_descriptor *) xor;
330 descs[1] = (struct ioat_raw_descriptor *) xor_ex;
331 for (i = 0; i < src_cnt; i++) {
332 dma_addr_t src = xor_get_src(descs, i);
333
334 ioat_unmap(pdev, src - offset, len,
335 PCI_DMA_TODEVICE, flags, 0);
336 }
337
338 /* dest is a source in xor validate operations */
339 if (xor->ctl_f.op == IOAT_OP_XOR_VAL) {
340 ioat_unmap(pdev, xor->dst_addr - offset, len,
341 PCI_DMA_TODEVICE, flags, 1);
342 break;
343 }
344 }
345
346 if (!(flags & DMA_COMPL_SKIP_DEST_UNMAP))
347 ioat_unmap(pdev, xor->dst_addr - offset, len,
348 PCI_DMA_FROMDEVICE, flags, 1);
349 break;
350 }
351 case IOAT_OP_PQ_VAL:
352 case IOAT_OP_PQ: {
353 struct ioat_pq_descriptor *pq = desc->pq;
354 struct ioat_ring_ent *ext;
355 struct ioat_pq_ext_descriptor *pq_ex = NULL;
356 int src_cnt = src_cnt_to_sw(pq->ctl_f.src_cnt);
357 struct ioat_raw_descriptor *descs[2];
358 int i;
359
360 if (src_cnt > 3) {
361 ext = ioat2_get_ring_ent(ioat, idx + 1);
362 pq_ex = ext->pq_ex;
363 }
364
365 /* in the 'continue' case don't unmap the dests as sources */
366 if (dmaf_p_disabled_continue(flags))
367 src_cnt--;
368 else if (dmaf_continue(flags))
369 src_cnt -= 3;
370
371 if (!(flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
372 descs[0] = (struct ioat_raw_descriptor *) pq;
373 descs[1] = (struct ioat_raw_descriptor *) pq_ex;
374 for (i = 0; i < src_cnt; i++) {
375 dma_addr_t src = pq_get_src(descs, i);
376
377 ioat_unmap(pdev, src - offset, len,
378 PCI_DMA_TODEVICE, flags, 0);
379 }
380
381 /* the dests are sources in pq validate operations */
382 if (pq->ctl_f.op == IOAT_OP_XOR_VAL) {
383 if (!(flags & DMA_PREP_PQ_DISABLE_P))
384 ioat_unmap(pdev, pq->p_addr - offset,
385 len, PCI_DMA_TODEVICE, flags, 0);
386 if (!(flags & DMA_PREP_PQ_DISABLE_Q))
387 ioat_unmap(pdev, pq->q_addr - offset,
388 len, PCI_DMA_TODEVICE, flags, 0);
389 break;
390 }
391 }
392
393 if (!(flags & DMA_COMPL_SKIP_DEST_UNMAP)) {
394 if (!(flags & DMA_PREP_PQ_DISABLE_P))
395 ioat_unmap(pdev, pq->p_addr - offset, len,
396 PCI_DMA_BIDIRECTIONAL, flags, 1);
397 if (!(flags & DMA_PREP_PQ_DISABLE_Q))
398 ioat_unmap(pdev, pq->q_addr - offset, len,
399 PCI_DMA_BIDIRECTIONAL, flags, 1);
400 }
401 break;
402 }
403 case IOAT_OP_PQ_16S:
404 case IOAT_OP_PQ_VAL_16S: {
405 struct ioat_pq_descriptor *pq = desc->pq;
406 int src_cnt = src16_cnt_to_sw(pq->ctl_f.src_cnt);
407 struct ioat_raw_descriptor *descs[4];
408 int i;
409
410 /* in the 'continue' case don't unmap the dests as sources */
411 if (dmaf_p_disabled_continue(flags))
412 src_cnt--;
413 else if (dmaf_continue(flags))
414 src_cnt -= 3;
415
416 if (!(flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
417 descs[0] = (struct ioat_raw_descriptor *)pq;
418 descs[1] = (struct ioat_raw_descriptor *)(desc->sed->hw);
419 descs[2] = (struct ioat_raw_descriptor *)(&desc->sed->hw->b[0]);
420 for (i = 0; i < src_cnt; i++) {
421 dma_addr_t src = pq16_get_src(descs, i);
422
423 ioat_unmap(pdev, src - offset, len,
424 PCI_DMA_TODEVICE, flags, 0);
425 }
426
427 /* the dests are sources in pq validate operations */
428 if (pq->ctl_f.op == IOAT_OP_XOR_VAL) {
429 if (!(flags & DMA_PREP_PQ_DISABLE_P))
430 ioat_unmap(pdev, pq->p_addr - offset,
431 len, PCI_DMA_TODEVICE,
432 flags, 0);
433 if (!(flags & DMA_PREP_PQ_DISABLE_Q))
434 ioat_unmap(pdev, pq->q_addr - offset,
435 len, PCI_DMA_TODEVICE,
436 flags, 0);
437 break;
438 }
439 }
440
441 if (!(flags & DMA_COMPL_SKIP_DEST_UNMAP)) {
442 if (!(flags & DMA_PREP_PQ_DISABLE_P))
443 ioat_unmap(pdev, pq->p_addr - offset, len,
444 PCI_DMA_BIDIRECTIONAL, flags, 1);
445 if (!(flags & DMA_PREP_PQ_DISABLE_Q))
446 ioat_unmap(pdev, pq->q_addr - offset, len,
447 PCI_DMA_BIDIRECTIONAL, flags, 1);
448 }
449 break;
450 }
451 default:
452 dev_err(&pdev->dev, "%s: unknown op type: %#x\n",
453 __func__, desc->hw->ctl_f.op);
454 }
455} 279}
456 280
457static bool desc_has_ext(struct ioat_ring_ent *desc) 281static bool desc_has_ext(struct ioat_ring_ent *desc)
@@ -577,7 +401,7 @@ static void __cleanup(struct ioat2_dma_chan *ioat, dma_addr_t phys_complete)
577 tx = &desc->txd; 401 tx = &desc->txd;
578 if (tx->cookie) { 402 if (tx->cookie) {
579 dma_cookie_complete(tx); 403 dma_cookie_complete(tx);
580 ioat3_dma_unmap(ioat, desc, idx + i); 404 dma_descriptor_unmap(tx);
581 if (tx->callback) { 405 if (tx->callback) {
582 tx->callback(tx->callback_param); 406 tx->callback(tx->callback_param);
583 tx->callback = NULL; 407 tx->callback = NULL;
@@ -1129,9 +953,6 @@ __ioat3_prep_pq16_lock(struct dma_chan *c, enum sum_check_flags *result,
1129 u8 op; 953 u8 op;
1130 int i, s, idx, num_descs; 954 int i, s, idx, num_descs;
1131 955
1132 /* this function only handles src_cnt 9 - 16 */
1133 BUG_ON(src_cnt < 9);
1134
1135 /* this function is only called with 9-16 sources */ 956 /* this function is only called with 9-16 sources */
1136 op = result ? IOAT_OP_PQ_VAL_16S : IOAT_OP_PQ_16S; 957 op = result ? IOAT_OP_PQ_VAL_16S : IOAT_OP_PQ_16S;
1137 958
@@ -1159,8 +980,7 @@ __ioat3_prep_pq16_lock(struct dma_chan *c, enum sum_check_flags *result,
1159 980
1160 descs[0] = (struct ioat_raw_descriptor *) pq; 981 descs[0] = (struct ioat_raw_descriptor *) pq;
1161 982
1162 desc->sed = ioat3_alloc_sed(device, 983 desc->sed = ioat3_alloc_sed(device, (src_cnt-2) >> 3);
1163 sed_get_pq16_pool_idx(src_cnt));
1164 if (!desc->sed) { 984 if (!desc->sed) {
1165 dev_err(to_dev(chan), 985 dev_err(to_dev(chan),
1166 "%s: no free sed entries\n", __func__); 986 "%s: no free sed entries\n", __func__);
@@ -1218,13 +1038,21 @@ __ioat3_prep_pq16_lock(struct dma_chan *c, enum sum_check_flags *result,
1218 return &desc->txd; 1038 return &desc->txd;
1219} 1039}
1220 1040
1041static int src_cnt_flags(unsigned int src_cnt, unsigned long flags)
1042{
1043 if (dmaf_p_disabled_continue(flags))
1044 return src_cnt + 1;
1045 else if (dmaf_continue(flags))
1046 return src_cnt + 3;
1047 else
1048 return src_cnt;
1049}
1050
1221static struct dma_async_tx_descriptor * 1051static struct dma_async_tx_descriptor *
1222ioat3_prep_pq(struct dma_chan *chan, dma_addr_t *dst, dma_addr_t *src, 1052ioat3_prep_pq(struct dma_chan *chan, dma_addr_t *dst, dma_addr_t *src,
1223 unsigned int src_cnt, const unsigned char *scf, size_t len, 1053 unsigned int src_cnt, const unsigned char *scf, size_t len,
1224 unsigned long flags) 1054 unsigned long flags)
1225{ 1055{
1226 struct dma_device *dma = chan->device;
1227
1228 /* specify valid address for disabled result */ 1056 /* specify valid address for disabled result */
1229 if (flags & DMA_PREP_PQ_DISABLE_P) 1057 if (flags & DMA_PREP_PQ_DISABLE_P)
1230 dst[0] = dst[1]; 1058 dst[0] = dst[1];
@@ -1244,7 +1072,7 @@ ioat3_prep_pq(struct dma_chan *chan, dma_addr_t *dst, dma_addr_t *src,
1244 single_source_coef[0] = scf[0]; 1072 single_source_coef[0] = scf[0];
1245 single_source_coef[1] = 0; 1073 single_source_coef[1] = 0;
1246 1074
1247 return (src_cnt > 8) && (dma->max_pq > 8) ? 1075 return src_cnt_flags(src_cnt, flags) > 8 ?
1248 __ioat3_prep_pq16_lock(chan, NULL, dst, single_source, 1076 __ioat3_prep_pq16_lock(chan, NULL, dst, single_source,
1249 2, single_source_coef, len, 1077 2, single_source_coef, len,
1250 flags) : 1078 flags) :
@@ -1252,7 +1080,7 @@ ioat3_prep_pq(struct dma_chan *chan, dma_addr_t *dst, dma_addr_t *src,
1252 single_source_coef, len, flags); 1080 single_source_coef, len, flags);
1253 1081
1254 } else { 1082 } else {
1255 return (src_cnt > 8) && (dma->max_pq > 8) ? 1083 return src_cnt_flags(src_cnt, flags) > 8 ?
1256 __ioat3_prep_pq16_lock(chan, NULL, dst, src, src_cnt, 1084 __ioat3_prep_pq16_lock(chan, NULL, dst, src, src_cnt,
1257 scf, len, flags) : 1085 scf, len, flags) :
1258 __ioat3_prep_pq_lock(chan, NULL, dst, src, src_cnt, 1086 __ioat3_prep_pq_lock(chan, NULL, dst, src, src_cnt,
@@ -1265,8 +1093,6 @@ ioat3_prep_pq_val(struct dma_chan *chan, dma_addr_t *pq, dma_addr_t *src,
1265 unsigned int src_cnt, const unsigned char *scf, size_t len, 1093 unsigned int src_cnt, const unsigned char *scf, size_t len,
1266 enum sum_check_flags *pqres, unsigned long flags) 1094 enum sum_check_flags *pqres, unsigned long flags)
1267{ 1095{
1268 struct dma_device *dma = chan->device;
1269
1270 /* specify valid address for disabled result */ 1096 /* specify valid address for disabled result */
1271 if (flags & DMA_PREP_PQ_DISABLE_P) 1097 if (flags & DMA_PREP_PQ_DISABLE_P)
1272 pq[0] = pq[1]; 1098 pq[0] = pq[1];
@@ -1278,7 +1104,7 @@ ioat3_prep_pq_val(struct dma_chan *chan, dma_addr_t *pq, dma_addr_t *src,
1278 */ 1104 */
1279 *pqres = 0; 1105 *pqres = 0;
1280 1106
1281 return (src_cnt > 8) && (dma->max_pq > 8) ? 1107 return src_cnt_flags(src_cnt, flags) > 8 ?
1282 __ioat3_prep_pq16_lock(chan, pqres, pq, src, src_cnt, scf, len, 1108 __ioat3_prep_pq16_lock(chan, pqres, pq, src, src_cnt, scf, len,
1283 flags) : 1109 flags) :
1284 __ioat3_prep_pq_lock(chan, pqres, pq, src, src_cnt, scf, len, 1110 __ioat3_prep_pq_lock(chan, pqres, pq, src, src_cnt, scf, len,
@@ -1289,7 +1115,6 @@ static struct dma_async_tx_descriptor *
1289ioat3_prep_pqxor(struct dma_chan *chan, dma_addr_t dst, dma_addr_t *src, 1115ioat3_prep_pqxor(struct dma_chan *chan, dma_addr_t dst, dma_addr_t *src,
1290 unsigned int src_cnt, size_t len, unsigned long flags) 1116 unsigned int src_cnt, size_t len, unsigned long flags)
1291{ 1117{
1292 struct dma_device *dma = chan->device;
1293 unsigned char scf[src_cnt]; 1118 unsigned char scf[src_cnt];
1294 dma_addr_t pq[2]; 1119 dma_addr_t pq[2];
1295 1120
@@ -1298,7 +1123,7 @@ ioat3_prep_pqxor(struct dma_chan *chan, dma_addr_t dst, dma_addr_t *src,
1298 flags |= DMA_PREP_PQ_DISABLE_Q; 1123 flags |= DMA_PREP_PQ_DISABLE_Q;
1299 pq[1] = dst; /* specify valid address for disabled result */ 1124 pq[1] = dst; /* specify valid address for disabled result */
1300 1125
1301 return (src_cnt > 8) && (dma->max_pq > 8) ? 1126 return src_cnt_flags(src_cnt, flags) > 8 ?
1302 __ioat3_prep_pq16_lock(chan, NULL, pq, src, src_cnt, scf, len, 1127 __ioat3_prep_pq16_lock(chan, NULL, pq, src, src_cnt, scf, len,
1303 flags) : 1128 flags) :
1304 __ioat3_prep_pq_lock(chan, NULL, pq, src, src_cnt, scf, len, 1129 __ioat3_prep_pq_lock(chan, NULL, pq, src, src_cnt, scf, len,
@@ -1310,7 +1135,6 @@ ioat3_prep_pqxor_val(struct dma_chan *chan, dma_addr_t *src,
1310 unsigned int src_cnt, size_t len, 1135 unsigned int src_cnt, size_t len,
1311 enum sum_check_flags *result, unsigned long flags) 1136 enum sum_check_flags *result, unsigned long flags)
1312{ 1137{
1313 struct dma_device *dma = chan->device;
1314 unsigned char scf[src_cnt]; 1138 unsigned char scf[src_cnt];
1315 dma_addr_t pq[2]; 1139 dma_addr_t pq[2];
1316 1140
@@ -1324,8 +1148,7 @@ ioat3_prep_pqxor_val(struct dma_chan *chan, dma_addr_t *src,
1324 flags |= DMA_PREP_PQ_DISABLE_Q; 1148 flags |= DMA_PREP_PQ_DISABLE_Q;
1325 pq[1] = pq[0]; /* specify valid address for disabled result */ 1149 pq[1] = pq[0]; /* specify valid address for disabled result */
1326 1150
1327 1151 return src_cnt_flags(src_cnt, flags) > 8 ?
1328 return (src_cnt > 8) && (dma->max_pq > 8) ?
1329 __ioat3_prep_pq16_lock(chan, result, pq, &src[1], src_cnt - 1, 1152 __ioat3_prep_pq16_lock(chan, result, pq, &src[1], src_cnt - 1,
1330 scf, len, flags) : 1153 scf, len, flags) :
1331 __ioat3_prep_pq_lock(chan, result, pq, &src[1], src_cnt - 1, 1154 __ioat3_prep_pq_lock(chan, result, pq, &src[1], src_cnt - 1,
@@ -1444,9 +1267,7 @@ static int ioat_xor_val_self_test(struct ioatdma_device *device)
1444 DMA_TO_DEVICE); 1267 DMA_TO_DEVICE);
1445 tx = dma->device_prep_dma_xor(dma_chan, dest_dma, dma_srcs, 1268 tx = dma->device_prep_dma_xor(dma_chan, dest_dma, dma_srcs,
1446 IOAT_NUM_SRC_TEST, PAGE_SIZE, 1269 IOAT_NUM_SRC_TEST, PAGE_SIZE,
1447 DMA_PREP_INTERRUPT | 1270 DMA_PREP_INTERRUPT);
1448 DMA_COMPL_SKIP_SRC_UNMAP |
1449 DMA_COMPL_SKIP_DEST_UNMAP);
1450 1271
1451 if (!tx) { 1272 if (!tx) {
1452 dev_err(dev, "Self-test xor prep failed\n"); 1273 dev_err(dev, "Self-test xor prep failed\n");
@@ -1507,9 +1328,7 @@ static int ioat_xor_val_self_test(struct ioatdma_device *device)
1507 DMA_TO_DEVICE); 1328 DMA_TO_DEVICE);
1508 tx = dma->device_prep_dma_xor_val(dma_chan, dma_srcs, 1329 tx = dma->device_prep_dma_xor_val(dma_chan, dma_srcs,
1509 IOAT_NUM_SRC_TEST + 1, PAGE_SIZE, 1330 IOAT_NUM_SRC_TEST + 1, PAGE_SIZE,
1510 &xor_val_result, DMA_PREP_INTERRUPT | 1331 &xor_val_result, DMA_PREP_INTERRUPT);
1511 DMA_COMPL_SKIP_SRC_UNMAP |
1512 DMA_COMPL_SKIP_DEST_UNMAP);
1513 if (!tx) { 1332 if (!tx) {
1514 dev_err(dev, "Self-test zero prep failed\n"); 1333 dev_err(dev, "Self-test zero prep failed\n");
1515 err = -ENODEV; 1334 err = -ENODEV;
@@ -1545,6 +1364,8 @@ static int ioat_xor_val_self_test(struct ioatdma_device *device)
1545 goto free_resources; 1364 goto free_resources;
1546 } 1365 }
1547 1366
1367 memset(page_address(dest), 0, PAGE_SIZE);
1368
1548 /* test for non-zero parity sum */ 1369 /* test for non-zero parity sum */
1549 op = IOAT_OP_XOR_VAL; 1370 op = IOAT_OP_XOR_VAL;
1550 1371
@@ -1554,9 +1375,7 @@ static int ioat_xor_val_self_test(struct ioatdma_device *device)
1554 DMA_TO_DEVICE); 1375 DMA_TO_DEVICE);
1555 tx = dma->device_prep_dma_xor_val(dma_chan, dma_srcs, 1376 tx = dma->device_prep_dma_xor_val(dma_chan, dma_srcs,
1556 IOAT_NUM_SRC_TEST + 1, PAGE_SIZE, 1377 IOAT_NUM_SRC_TEST + 1, PAGE_SIZE,
1557 &xor_val_result, DMA_PREP_INTERRUPT | 1378 &xor_val_result, DMA_PREP_INTERRUPT);
1558 DMA_COMPL_SKIP_SRC_UNMAP |
1559 DMA_COMPL_SKIP_DEST_UNMAP);
1560 if (!tx) { 1379 if (!tx) {
1561 dev_err(dev, "Self-test 2nd zero prep failed\n"); 1380 dev_err(dev, "Self-test 2nd zero prep failed\n");
1562 err = -ENODEV; 1381 err = -ENODEV;
@@ -1630,52 +1449,36 @@ static int ioat3_dma_self_test(struct ioatdma_device *device)
1630 1449
1631static int ioat3_irq_reinit(struct ioatdma_device *device) 1450static int ioat3_irq_reinit(struct ioatdma_device *device)
1632{ 1451{
1633 int msixcnt = device->common.chancnt;
1634 struct pci_dev *pdev = device->pdev; 1452 struct pci_dev *pdev = device->pdev;
1635 int i; 1453 int irq = pdev->irq, i;
1636 struct msix_entry *msix; 1454
1637 struct ioat_chan_common *chan; 1455 if (!is_bwd_ioat(pdev))
1638 int err = 0; 1456 return 0;
1639 1457
1640 switch (device->irq_mode) { 1458 switch (device->irq_mode) {
1641 case IOAT_MSIX: 1459 case IOAT_MSIX:
1460 for (i = 0; i < device->common.chancnt; i++) {
1461 struct msix_entry *msix = &device->msix_entries[i];
1462 struct ioat_chan_common *chan;
1642 1463
1643 for (i = 0; i < msixcnt; i++) {
1644 msix = &device->msix_entries[i];
1645 chan = ioat_chan_by_index(device, i); 1464 chan = ioat_chan_by_index(device, i);
1646 devm_free_irq(&pdev->dev, msix->vector, chan); 1465 devm_free_irq(&pdev->dev, msix->vector, chan);
1647 } 1466 }
1648 1467
1649 pci_disable_msix(pdev); 1468 pci_disable_msix(pdev);
1650 break; 1469 break;
1651
1652 case IOAT_MSIX_SINGLE:
1653 msix = &device->msix_entries[0];
1654 chan = ioat_chan_by_index(device, 0);
1655 devm_free_irq(&pdev->dev, msix->vector, chan);
1656 pci_disable_msix(pdev);
1657 break;
1658
1659 case IOAT_MSI: 1470 case IOAT_MSI:
1660 chan = ioat_chan_by_index(device, 0);
1661 devm_free_irq(&pdev->dev, pdev->irq, chan);
1662 pci_disable_msi(pdev); 1471 pci_disable_msi(pdev);
1663 break; 1472 /* fall through */
1664
1665 case IOAT_INTX: 1473 case IOAT_INTX:
1666 chan = ioat_chan_by_index(device, 0); 1474 devm_free_irq(&pdev->dev, irq, device);
1667 devm_free_irq(&pdev->dev, pdev->irq, chan);
1668 break; 1475 break;
1669
1670 default: 1476 default:
1671 return 0; 1477 return 0;
1672 } 1478 }
1673
1674 device->irq_mode = IOAT_NOIRQ; 1479 device->irq_mode = IOAT_NOIRQ;
1675 1480
1676 err = ioat_dma_setup_interrupts(device); 1481 return ioat_dma_setup_interrupts(device);
1677
1678 return err;
1679} 1482}
1680 1483
1681static int ioat3_reset_hw(struct ioat_chan_common *chan) 1484static int ioat3_reset_hw(struct ioat_chan_common *chan)
@@ -1718,14 +1521,12 @@ static int ioat3_reset_hw(struct ioat_chan_common *chan)
1718 } 1521 }
1719 1522
1720 err = ioat2_reset_sync(chan, msecs_to_jiffies(200)); 1523 err = ioat2_reset_sync(chan, msecs_to_jiffies(200));
1721 if (err) { 1524 if (!err)
1722 dev_err(&pdev->dev, "Failed to reset!\n");
1723 return err;
1724 }
1725
1726 if (device->irq_mode != IOAT_NOIRQ && is_bwd_ioat(pdev))
1727 err = ioat3_irq_reinit(device); 1525 err = ioat3_irq_reinit(device);
1728 1526
1527 if (err)
1528 dev_err(&pdev->dev, "Failed to reset: %d\n", err);
1529
1729 return err; 1530 return err;
1730} 1531}
1731 1532
@@ -1835,21 +1636,15 @@ int ioat3_dma_probe(struct ioatdma_device *device, int dca)
1835 char pool_name[14]; 1636 char pool_name[14];
1836 int i; 1637 int i;
1837 1638
1838 /* allocate sw descriptor pool for SED */
1839 device->sed_pool = kmem_cache_create("ioat_sed",
1840 sizeof(struct ioat_sed_ent), 0, 0, NULL);
1841 if (!device->sed_pool)
1842 return -ENOMEM;
1843
1844 for (i = 0; i < MAX_SED_POOLS; i++) { 1639 for (i = 0; i < MAX_SED_POOLS; i++) {
1845 snprintf(pool_name, 14, "ioat_hw%d_sed", i); 1640 snprintf(pool_name, 14, "ioat_hw%d_sed", i);
1846 1641
1847 /* allocate SED DMA pool */ 1642 /* allocate SED DMA pool */
1848 device->sed_hw_pool[i] = dma_pool_create(pool_name, 1643 device->sed_hw_pool[i] = dmam_pool_create(pool_name,
1849 &pdev->dev, 1644 &pdev->dev,
1850 SED_SIZE * (i + 1), 64, 0); 1645 SED_SIZE * (i + 1), 64, 0);
1851 if (!device->sed_hw_pool[i]) 1646 if (!device->sed_hw_pool[i])
1852 goto sed_pool_cleanup; 1647 return -ENOMEM;
1853 1648
1854 } 1649 }
1855 } 1650 }
@@ -1875,28 +1670,4 @@ int ioat3_dma_probe(struct ioatdma_device *device, int dca)
1875 device->dca = ioat3_dca_init(pdev, device->reg_base); 1670 device->dca = ioat3_dca_init(pdev, device->reg_base);
1876 1671
1877 return 0; 1672 return 0;
1878
1879sed_pool_cleanup:
1880 if (device->sed_pool) {
1881 int i;
1882 kmem_cache_destroy(device->sed_pool);
1883
1884 for (i = 0; i < MAX_SED_POOLS; i++)
1885 if (device->sed_hw_pool[i])
1886 dma_pool_destroy(device->sed_hw_pool[i]);
1887 }
1888
1889 return -ENOMEM;
1890}
1891
1892void ioat3_dma_remove(struct ioatdma_device *device)
1893{
1894 if (device->sed_pool) {
1895 int i;
1896 kmem_cache_destroy(device->sed_pool);
1897
1898 for (i = 0; i < MAX_SED_POOLS; i++)
1899 if (device->sed_hw_pool[i])
1900 dma_pool_destroy(device->sed_hw_pool[i]);
1901 }
1902} 1673}
diff --git a/drivers/dma/ioat/pci.c b/drivers/dma/ioat/pci.c
index 2c8d560e6334..1d051cd045db 100644
--- a/drivers/dma/ioat/pci.c
+++ b/drivers/dma/ioat/pci.c
@@ -123,6 +123,7 @@ module_param(ioat_dca_enabled, int, 0644);
123MODULE_PARM_DESC(ioat_dca_enabled, "control support of dca service (default: 1)"); 123MODULE_PARM_DESC(ioat_dca_enabled, "control support of dca service (default: 1)");
124 124
125struct kmem_cache *ioat2_cache; 125struct kmem_cache *ioat2_cache;
126struct kmem_cache *ioat3_sed_cache;
126 127
127#define DRV_NAME "ioatdma" 128#define DRV_NAME "ioatdma"
128 129
@@ -207,9 +208,6 @@ static void ioat_remove(struct pci_dev *pdev)
207 if (!device) 208 if (!device)
208 return; 209 return;
209 210
210 if (device->version >= IOAT_VER_3_0)
211 ioat3_dma_remove(device);
212
213 dev_err(&pdev->dev, "Removing dma and dca services\n"); 211 dev_err(&pdev->dev, "Removing dma and dca services\n");
214 if (device->dca) { 212 if (device->dca) {
215 unregister_dca_provider(device->dca, &pdev->dev); 213 unregister_dca_provider(device->dca, &pdev->dev);
@@ -221,7 +219,7 @@ static void ioat_remove(struct pci_dev *pdev)
221 219
222static int __init ioat_init_module(void) 220static int __init ioat_init_module(void)
223{ 221{
224 int err; 222 int err = -ENOMEM;
225 223
226 pr_info("%s: Intel(R) QuickData Technology Driver %s\n", 224 pr_info("%s: Intel(R) QuickData Technology Driver %s\n",
227 DRV_NAME, IOAT_DMA_VERSION); 225 DRV_NAME, IOAT_DMA_VERSION);
@@ -231,9 +229,21 @@ static int __init ioat_init_module(void)
231 if (!ioat2_cache) 229 if (!ioat2_cache)
232 return -ENOMEM; 230 return -ENOMEM;
233 231
232 ioat3_sed_cache = KMEM_CACHE(ioat_sed_ent, 0);
233 if (!ioat3_sed_cache)
234 goto err_ioat2_cache;
235
234 err = pci_register_driver(&ioat_pci_driver); 236 err = pci_register_driver(&ioat_pci_driver);
235 if (err) 237 if (err)
236 kmem_cache_destroy(ioat2_cache); 238 goto err_ioat3_cache;
239
240 return 0;
241
242 err_ioat3_cache:
243 kmem_cache_destroy(ioat3_sed_cache);
244
245 err_ioat2_cache:
246 kmem_cache_destroy(ioat2_cache);
237 247
238 return err; 248 return err;
239} 249}
diff --git a/drivers/dma/iop-adma.c b/drivers/dma/iop-adma.c
index 408fe6be15f4..c56137bc3868 100644
--- a/drivers/dma/iop-adma.c
+++ b/drivers/dma/iop-adma.c
@@ -61,80 +61,6 @@ static void iop_adma_free_slots(struct iop_adma_desc_slot *slot)
61 } 61 }
62} 62}
63 63
64static void
65iop_desc_unmap(struct iop_adma_chan *iop_chan, struct iop_adma_desc_slot *desc)
66{
67 struct dma_async_tx_descriptor *tx = &desc->async_tx;
68 struct iop_adma_desc_slot *unmap = desc->group_head;
69 struct device *dev = &iop_chan->device->pdev->dev;
70 u32 len = unmap->unmap_len;
71 enum dma_ctrl_flags flags = tx->flags;
72 u32 src_cnt;
73 dma_addr_t addr;
74 dma_addr_t dest;
75
76 src_cnt = unmap->unmap_src_cnt;
77 dest = iop_desc_get_dest_addr(unmap, iop_chan);
78 if (!(flags & DMA_COMPL_SKIP_DEST_UNMAP)) {
79 enum dma_data_direction dir;
80
81 if (src_cnt > 1) /* is xor? */
82 dir = DMA_BIDIRECTIONAL;
83 else
84 dir = DMA_FROM_DEVICE;
85
86 dma_unmap_page(dev, dest, len, dir);
87 }
88
89 if (!(flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
90 while (src_cnt--) {
91 addr = iop_desc_get_src_addr(unmap, iop_chan, src_cnt);
92 if (addr == dest)
93 continue;
94 dma_unmap_page(dev, addr, len, DMA_TO_DEVICE);
95 }
96 }
97 desc->group_head = NULL;
98}
99
100static void
101iop_desc_unmap_pq(struct iop_adma_chan *iop_chan, struct iop_adma_desc_slot *desc)
102{
103 struct dma_async_tx_descriptor *tx = &desc->async_tx;
104 struct iop_adma_desc_slot *unmap = desc->group_head;
105 struct device *dev = &iop_chan->device->pdev->dev;
106 u32 len = unmap->unmap_len;
107 enum dma_ctrl_flags flags = tx->flags;
108 u32 src_cnt = unmap->unmap_src_cnt;
109 dma_addr_t pdest = iop_desc_get_dest_addr(unmap, iop_chan);
110 dma_addr_t qdest = iop_desc_get_qdest_addr(unmap, iop_chan);
111 int i;
112
113 if (tx->flags & DMA_PREP_CONTINUE)
114 src_cnt -= 3;
115
116 if (!(flags & DMA_COMPL_SKIP_DEST_UNMAP) && !desc->pq_check_result) {
117 dma_unmap_page(dev, pdest, len, DMA_BIDIRECTIONAL);
118 dma_unmap_page(dev, qdest, len, DMA_BIDIRECTIONAL);
119 }
120
121 if (!(flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
122 dma_addr_t addr;
123
124 for (i = 0; i < src_cnt; i++) {
125 addr = iop_desc_get_src_addr(unmap, iop_chan, i);
126 dma_unmap_page(dev, addr, len, DMA_TO_DEVICE);
127 }
128 if (desc->pq_check_result) {
129 dma_unmap_page(dev, pdest, len, DMA_TO_DEVICE);
130 dma_unmap_page(dev, qdest, len, DMA_TO_DEVICE);
131 }
132 }
133
134 desc->group_head = NULL;
135}
136
137
138static dma_cookie_t 64static dma_cookie_t
139iop_adma_run_tx_complete_actions(struct iop_adma_desc_slot *desc, 65iop_adma_run_tx_complete_actions(struct iop_adma_desc_slot *desc,
140 struct iop_adma_chan *iop_chan, dma_cookie_t cookie) 66 struct iop_adma_chan *iop_chan, dma_cookie_t cookie)
@@ -152,15 +78,9 @@ iop_adma_run_tx_complete_actions(struct iop_adma_desc_slot *desc,
152 if (tx->callback) 78 if (tx->callback)
153 tx->callback(tx->callback_param); 79 tx->callback(tx->callback_param);
154 80
155 /* unmap dma addresses 81 dma_descriptor_unmap(tx);
156 * (unmap_single vs unmap_page?) 82 if (desc->group_head)
157 */ 83 desc->group_head = NULL;
158 if (desc->group_head && desc->unmap_len) {
159 if (iop_desc_is_pq(desc))
160 iop_desc_unmap_pq(iop_chan, desc);
161 else
162 iop_desc_unmap(iop_chan, desc);
163 }
164 } 84 }
165 85
166 /* run dependent operations */ 86 /* run dependent operations */
@@ -591,7 +511,6 @@ iop_adma_prep_dma_interrupt(struct dma_chan *chan, unsigned long flags)
591 if (sw_desc) { 511 if (sw_desc) {
592 grp_start = sw_desc->group_head; 512 grp_start = sw_desc->group_head;
593 iop_desc_init_interrupt(grp_start, iop_chan); 513 iop_desc_init_interrupt(grp_start, iop_chan);
594 grp_start->unmap_len = 0;
595 sw_desc->async_tx.flags = flags; 514 sw_desc->async_tx.flags = flags;
596 } 515 }
597 spin_unlock_bh(&iop_chan->lock); 516 spin_unlock_bh(&iop_chan->lock);
@@ -623,8 +542,6 @@ iop_adma_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dma_dest,
623 iop_desc_set_byte_count(grp_start, iop_chan, len); 542 iop_desc_set_byte_count(grp_start, iop_chan, len);
624 iop_desc_set_dest_addr(grp_start, iop_chan, dma_dest); 543 iop_desc_set_dest_addr(grp_start, iop_chan, dma_dest);
625 iop_desc_set_memcpy_src_addr(grp_start, dma_src); 544 iop_desc_set_memcpy_src_addr(grp_start, dma_src);
626 sw_desc->unmap_src_cnt = 1;
627 sw_desc->unmap_len = len;
628 sw_desc->async_tx.flags = flags; 545 sw_desc->async_tx.flags = flags;
629 } 546 }
630 spin_unlock_bh(&iop_chan->lock); 547 spin_unlock_bh(&iop_chan->lock);
@@ -657,8 +574,6 @@ iop_adma_prep_dma_xor(struct dma_chan *chan, dma_addr_t dma_dest,
657 iop_desc_init_xor(grp_start, src_cnt, flags); 574 iop_desc_init_xor(grp_start, src_cnt, flags);
658 iop_desc_set_byte_count(grp_start, iop_chan, len); 575 iop_desc_set_byte_count(grp_start, iop_chan, len);
659 iop_desc_set_dest_addr(grp_start, iop_chan, dma_dest); 576 iop_desc_set_dest_addr(grp_start, iop_chan, dma_dest);
660 sw_desc->unmap_src_cnt = src_cnt;
661 sw_desc->unmap_len = len;
662 sw_desc->async_tx.flags = flags; 577 sw_desc->async_tx.flags = flags;
663 while (src_cnt--) 578 while (src_cnt--)
664 iop_desc_set_xor_src_addr(grp_start, src_cnt, 579 iop_desc_set_xor_src_addr(grp_start, src_cnt,
@@ -694,8 +609,6 @@ iop_adma_prep_dma_xor_val(struct dma_chan *chan, dma_addr_t *dma_src,
694 grp_start->xor_check_result = result; 609 grp_start->xor_check_result = result;
695 pr_debug("\t%s: grp_start->xor_check_result: %p\n", 610 pr_debug("\t%s: grp_start->xor_check_result: %p\n",
696 __func__, grp_start->xor_check_result); 611 __func__, grp_start->xor_check_result);
697 sw_desc->unmap_src_cnt = src_cnt;
698 sw_desc->unmap_len = len;
699 sw_desc->async_tx.flags = flags; 612 sw_desc->async_tx.flags = flags;
700 while (src_cnt--) 613 while (src_cnt--)
701 iop_desc_set_zero_sum_src_addr(grp_start, src_cnt, 614 iop_desc_set_zero_sum_src_addr(grp_start, src_cnt,
@@ -748,8 +661,6 @@ iop_adma_prep_dma_pq(struct dma_chan *chan, dma_addr_t *dst, dma_addr_t *src,
748 dst[0] = dst[1] & 0x7; 661 dst[0] = dst[1] & 0x7;
749 662
750 iop_desc_set_pq_addr(g, dst); 663 iop_desc_set_pq_addr(g, dst);
751 sw_desc->unmap_src_cnt = src_cnt;
752 sw_desc->unmap_len = len;
753 sw_desc->async_tx.flags = flags; 664 sw_desc->async_tx.flags = flags;
754 for (i = 0; i < src_cnt; i++) 665 for (i = 0; i < src_cnt; i++)
755 iop_desc_set_pq_src_addr(g, i, src[i], scf[i]); 666 iop_desc_set_pq_src_addr(g, i, src[i], scf[i]);
@@ -804,8 +715,6 @@ iop_adma_prep_dma_pq_val(struct dma_chan *chan, dma_addr_t *pq, dma_addr_t *src,
804 g->pq_check_result = pqres; 715 g->pq_check_result = pqres;
805 pr_debug("\t%s: g->pq_check_result: %p\n", 716 pr_debug("\t%s: g->pq_check_result: %p\n",
806 __func__, g->pq_check_result); 717 __func__, g->pq_check_result);
807 sw_desc->unmap_src_cnt = src_cnt+2;
808 sw_desc->unmap_len = len;
809 sw_desc->async_tx.flags = flags; 718 sw_desc->async_tx.flags = flags;
810 while (src_cnt--) 719 while (src_cnt--)
811 iop_desc_set_pq_zero_sum_src_addr(g, src_cnt, 720 iop_desc_set_pq_zero_sum_src_addr(g, src_cnt,
diff --git a/drivers/dma/mv_xor.c b/drivers/dma/mv_xor.c
index 8d5bce9e867e..7807f0ef4e20 100644
--- a/drivers/dma/mv_xor.c
+++ b/drivers/dma/mv_xor.c
@@ -60,14 +60,6 @@ static u32 mv_desc_get_dest_addr(struct mv_xor_desc_slot *desc)
60 return hw_desc->phy_dest_addr; 60 return hw_desc->phy_dest_addr;
61} 61}
62 62
63static u32 mv_desc_get_src_addr(struct mv_xor_desc_slot *desc,
64 int src_idx)
65{
66 struct mv_xor_desc *hw_desc = desc->hw_desc;
67 return hw_desc->phy_src_addr[mv_phy_src_idx(src_idx)];
68}
69
70
71static void mv_desc_set_byte_count(struct mv_xor_desc_slot *desc, 63static void mv_desc_set_byte_count(struct mv_xor_desc_slot *desc,
72 u32 byte_count) 64 u32 byte_count)
73{ 65{
@@ -278,42 +270,9 @@ mv_xor_run_tx_complete_actions(struct mv_xor_desc_slot *desc,
278 desc->async_tx.callback( 270 desc->async_tx.callback(
279 desc->async_tx.callback_param); 271 desc->async_tx.callback_param);
280 272
281 /* unmap dma addresses 273 dma_descriptor_unmap(&desc->async_tx);
282 * (unmap_single vs unmap_page?) 274 if (desc->group_head)
283 */
284 if (desc->group_head && desc->unmap_len) {
285 struct mv_xor_desc_slot *unmap = desc->group_head;
286 struct device *dev = mv_chan_to_devp(mv_chan);
287 u32 len = unmap->unmap_len;
288 enum dma_ctrl_flags flags = desc->async_tx.flags;
289 u32 src_cnt;
290 dma_addr_t addr;
291 dma_addr_t dest;
292
293 src_cnt = unmap->unmap_src_cnt;
294 dest = mv_desc_get_dest_addr(unmap);
295 if (!(flags & DMA_COMPL_SKIP_DEST_UNMAP)) {
296 enum dma_data_direction dir;
297
298 if (src_cnt > 1) /* is xor ? */
299 dir = DMA_BIDIRECTIONAL;
300 else
301 dir = DMA_FROM_DEVICE;
302 dma_unmap_page(dev, dest, len, dir);
303 }
304
305 if (!(flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
306 while (src_cnt--) {
307 addr = mv_desc_get_src_addr(unmap,
308 src_cnt);
309 if (addr == dest)
310 continue;
311 dma_unmap_page(dev, addr, len,
312 DMA_TO_DEVICE);
313 }
314 }
315 desc->group_head = NULL; 275 desc->group_head = NULL;
316 }
317 } 276 }
318 277
319 /* run dependent operations */ 278 /* run dependent operations */
@@ -1076,10 +1035,7 @@ mv_xor_channel_add(struct mv_xor_device *xordev,
1076 } 1035 }
1077 1036
1078 mv_chan->mmr_base = xordev->xor_base; 1037 mv_chan->mmr_base = xordev->xor_base;
1079 if (!mv_chan->mmr_base) { 1038 mv_chan->mmr_high_base = xordev->xor_high_base;
1080 ret = -ENOMEM;
1081 goto err_free_dma;
1082 }
1083 tasklet_init(&mv_chan->irq_tasklet, mv_xor_tasklet, (unsigned long) 1039 tasklet_init(&mv_chan->irq_tasklet, mv_xor_tasklet, (unsigned long)
1084 mv_chan); 1040 mv_chan);
1085 1041
@@ -1138,7 +1094,7 @@ static void
1138mv_xor_conf_mbus_windows(struct mv_xor_device *xordev, 1094mv_xor_conf_mbus_windows(struct mv_xor_device *xordev,
1139 const struct mbus_dram_target_info *dram) 1095 const struct mbus_dram_target_info *dram)
1140{ 1096{
1141 void __iomem *base = xordev->xor_base; 1097 void __iomem *base = xordev->xor_high_base;
1142 u32 win_enable = 0; 1098 u32 win_enable = 0;
1143 int i; 1099 int i;
1144 1100
diff --git a/drivers/dma/mv_xor.h b/drivers/dma/mv_xor.h
index 06b067f24c9b..d0749229c875 100644
--- a/drivers/dma/mv_xor.h
+++ b/drivers/dma/mv_xor.h
@@ -34,13 +34,13 @@
34#define XOR_OPERATION_MODE_MEMCPY 2 34#define XOR_OPERATION_MODE_MEMCPY 2
35#define XOR_DESCRIPTOR_SWAP BIT(14) 35#define XOR_DESCRIPTOR_SWAP BIT(14)
36 36
37#define XOR_CURR_DESC(chan) (chan->mmr_base + 0x210 + (chan->idx * 4)) 37#define XOR_CURR_DESC(chan) (chan->mmr_high_base + 0x10 + (chan->idx * 4))
38#define XOR_NEXT_DESC(chan) (chan->mmr_base + 0x200 + (chan->idx * 4)) 38#define XOR_NEXT_DESC(chan) (chan->mmr_high_base + 0x00 + (chan->idx * 4))
39#define XOR_BYTE_COUNT(chan) (chan->mmr_base + 0x220 + (chan->idx * 4)) 39#define XOR_BYTE_COUNT(chan) (chan->mmr_high_base + 0x20 + (chan->idx * 4))
40#define XOR_DEST_POINTER(chan) (chan->mmr_base + 0x2B0 + (chan->idx * 4)) 40#define XOR_DEST_POINTER(chan) (chan->mmr_high_base + 0xB0 + (chan->idx * 4))
41#define XOR_BLOCK_SIZE(chan) (chan->mmr_base + 0x2C0 + (chan->idx * 4)) 41#define XOR_BLOCK_SIZE(chan) (chan->mmr_high_base + 0xC0 + (chan->idx * 4))
42#define XOR_INIT_VALUE_LOW(chan) (chan->mmr_base + 0x2E0) 42#define XOR_INIT_VALUE_LOW(chan) (chan->mmr_high_base + 0xE0)
43#define XOR_INIT_VALUE_HIGH(chan) (chan->mmr_base + 0x2E4) 43#define XOR_INIT_VALUE_HIGH(chan) (chan->mmr_high_base + 0xE4)
44 44
45#define XOR_CONFIG(chan) (chan->mmr_base + 0x10 + (chan->idx * 4)) 45#define XOR_CONFIG(chan) (chan->mmr_base + 0x10 + (chan->idx * 4))
46#define XOR_ACTIVATION(chan) (chan->mmr_base + 0x20 + (chan->idx * 4)) 46#define XOR_ACTIVATION(chan) (chan->mmr_base + 0x20 + (chan->idx * 4))
@@ -50,11 +50,11 @@
50#define XOR_ERROR_ADDR(chan) (chan->mmr_base + 0x60) 50#define XOR_ERROR_ADDR(chan) (chan->mmr_base + 0x60)
51#define XOR_INTR_MASK_VALUE 0x3F5 51#define XOR_INTR_MASK_VALUE 0x3F5
52 52
53#define WINDOW_BASE(w) (0x250 + ((w) << 2)) 53#define WINDOW_BASE(w) (0x50 + ((w) << 2))
54#define WINDOW_SIZE(w) (0x270 + ((w) << 2)) 54#define WINDOW_SIZE(w) (0x70 + ((w) << 2))
55#define WINDOW_REMAP_HIGH(w) (0x290 + ((w) << 2)) 55#define WINDOW_REMAP_HIGH(w) (0x90 + ((w) << 2))
56#define WINDOW_BAR_ENABLE(chan) (0x240 + ((chan) << 2)) 56#define WINDOW_BAR_ENABLE(chan) (0x40 + ((chan) << 2))
57#define WINDOW_OVERRIDE_CTRL(chan) (0x2A0 + ((chan) << 2)) 57#define WINDOW_OVERRIDE_CTRL(chan) (0xA0 + ((chan) << 2))
58 58
59struct mv_xor_device { 59struct mv_xor_device {
60 void __iomem *xor_base; 60 void __iomem *xor_base;
@@ -82,6 +82,7 @@ struct mv_xor_chan {
82 int pending; 82 int pending;
83 spinlock_t lock; /* protects the descriptor slot pool */ 83 spinlock_t lock; /* protects the descriptor slot pool */
84 void __iomem *mmr_base; 84 void __iomem *mmr_base;
85 void __iomem *mmr_high_base;
85 unsigned int idx; 86 unsigned int idx;
86 int irq; 87 int irq;
87 enum dma_transaction_type current_type; 88 enum dma_transaction_type current_type;
diff --git a/drivers/dma/pl330.c b/drivers/dma/pl330.c
index 0ee2c547b304..877d7bb767d9 100644
--- a/drivers/dma/pl330.c
+++ b/drivers/dma/pl330.c
@@ -2268,6 +2268,8 @@ static void pl330_tasklet(unsigned long data)
2268 list_move_tail(&desc->node, &pch->dmac->desc_pool); 2268 list_move_tail(&desc->node, &pch->dmac->desc_pool);
2269 } 2269 }
2270 2270
2271 dma_descriptor_unmap(&desc->txd);
2272
2271 if (callback) { 2273 if (callback) {
2272 spin_unlock_irqrestore(&pch->lock, flags); 2274 spin_unlock_irqrestore(&pch->lock, flags);
2273 callback(callback_param); 2275 callback(callback_param);
diff --git a/drivers/dma/ppc4xx/adma.c b/drivers/dma/ppc4xx/adma.c
index 60e02ae38b04..3005938ba47a 100644
--- a/drivers/dma/ppc4xx/adma.c
+++ b/drivers/dma/ppc4xx/adma.c
@@ -802,218 +802,6 @@ static void ppc440spe_desc_set_link(struct ppc440spe_adma_chan *chan,
802} 802}
803 803
804/** 804/**
805 * ppc440spe_desc_get_src_addr - extract the source address from the descriptor
806 */
807static u32 ppc440spe_desc_get_src_addr(struct ppc440spe_adma_desc_slot *desc,
808 struct ppc440spe_adma_chan *chan, int src_idx)
809{
810 struct dma_cdb *dma_hw_desc;
811 struct xor_cb *xor_hw_desc;
812
813 switch (chan->device->id) {
814 case PPC440SPE_DMA0_ID:
815 case PPC440SPE_DMA1_ID:
816 dma_hw_desc = desc->hw_desc;
817 /* May have 0, 1, 2, or 3 sources */
818 switch (dma_hw_desc->opc) {
819 case DMA_CDB_OPC_NO_OP:
820 case DMA_CDB_OPC_DFILL128:
821 return 0;
822 case DMA_CDB_OPC_DCHECK128:
823 if (unlikely(src_idx)) {
824 printk(KERN_ERR "%s: try to get %d source for"
825 " DCHECK128\n", __func__, src_idx);
826 BUG();
827 }
828 return le32_to_cpu(dma_hw_desc->sg1l);
829 case DMA_CDB_OPC_MULTICAST:
830 case DMA_CDB_OPC_MV_SG1_SG2:
831 if (unlikely(src_idx > 2)) {
832 printk(KERN_ERR "%s: try to get %d source from"
833 " DMA descr\n", __func__, src_idx);
834 BUG();
835 }
836 if (src_idx) {
837 if (le32_to_cpu(dma_hw_desc->sg1u) &
838 DMA_CUED_XOR_WIN_MSK) {
839 u8 region;
840
841 if (src_idx == 1)
842 return le32_to_cpu(
843 dma_hw_desc->sg1l) +
844 desc->unmap_len;
845
846 region = (le32_to_cpu(
847 dma_hw_desc->sg1u)) >>
848 DMA_CUED_REGION_OFF;
849
850 region &= DMA_CUED_REGION_MSK;
851 switch (region) {
852 case DMA_RXOR123:
853 return le32_to_cpu(
854 dma_hw_desc->sg1l) +
855 (desc->unmap_len << 1);
856 case DMA_RXOR124:
857 return le32_to_cpu(
858 dma_hw_desc->sg1l) +
859 (desc->unmap_len * 3);
860 case DMA_RXOR125:
861 return le32_to_cpu(
862 dma_hw_desc->sg1l) +
863 (desc->unmap_len << 2);
864 default:
865 printk(KERN_ERR
866 "%s: try to"
867 " get src3 for region %02x"
868 "PPC440SPE_DESC_RXOR12?\n",
869 __func__, region);
870 BUG();
871 }
872 } else {
873 printk(KERN_ERR
874 "%s: try to get %d"
875 " source for non-cued descr\n",
876 __func__, src_idx);
877 BUG();
878 }
879 }
880 return le32_to_cpu(dma_hw_desc->sg1l);
881 default:
882 printk(KERN_ERR "%s: unknown OPC 0x%02x\n",
883 __func__, dma_hw_desc->opc);
884 BUG();
885 }
886 return le32_to_cpu(dma_hw_desc->sg1l);
887 case PPC440SPE_XOR_ID:
888 /* May have up to 16 sources */
889 xor_hw_desc = desc->hw_desc;
890 return xor_hw_desc->ops[src_idx].l;
891 }
892 return 0;
893}
894
895/**
896 * ppc440spe_desc_get_dest_addr - extract the destination address from the
897 * descriptor
898 */
899static u32 ppc440spe_desc_get_dest_addr(struct ppc440spe_adma_desc_slot *desc,
900 struct ppc440spe_adma_chan *chan, int idx)
901{
902 struct dma_cdb *dma_hw_desc;
903 struct xor_cb *xor_hw_desc;
904
905 switch (chan->device->id) {
906 case PPC440SPE_DMA0_ID:
907 case PPC440SPE_DMA1_ID:
908 dma_hw_desc = desc->hw_desc;
909
910 if (likely(!idx))
911 return le32_to_cpu(dma_hw_desc->sg2l);
912 return le32_to_cpu(dma_hw_desc->sg3l);
913 case PPC440SPE_XOR_ID:
914 xor_hw_desc = desc->hw_desc;
915 return xor_hw_desc->cbtal;
916 }
917 return 0;
918}
919
920/**
921 * ppc440spe_desc_get_src_num - extract the number of source addresses from
922 * the descriptor
923 */
924static u32 ppc440spe_desc_get_src_num(struct ppc440spe_adma_desc_slot *desc,
925 struct ppc440spe_adma_chan *chan)
926{
927 struct dma_cdb *dma_hw_desc;
928 struct xor_cb *xor_hw_desc;
929
930 switch (chan->device->id) {
931 case PPC440SPE_DMA0_ID:
932 case PPC440SPE_DMA1_ID:
933 dma_hw_desc = desc->hw_desc;
934
935 switch (dma_hw_desc->opc) {
936 case DMA_CDB_OPC_NO_OP:
937 case DMA_CDB_OPC_DFILL128:
938 return 0;
939 case DMA_CDB_OPC_DCHECK128:
940 return 1;
941 case DMA_CDB_OPC_MV_SG1_SG2:
942 case DMA_CDB_OPC_MULTICAST:
943 /*
944 * Only for RXOR operations we have more than
945 * one source
946 */
947 if (le32_to_cpu(dma_hw_desc->sg1u) &
948 DMA_CUED_XOR_WIN_MSK) {
949 /* RXOR op, there are 2 or 3 sources */
950 if (((le32_to_cpu(dma_hw_desc->sg1u) >>
951 DMA_CUED_REGION_OFF) &
952 DMA_CUED_REGION_MSK) == DMA_RXOR12) {
953 /* RXOR 1-2 */
954 return 2;
955 } else {
956 /* RXOR 1-2-3/1-2-4/1-2-5 */
957 return 3;
958 }
959 }
960 return 1;
961 default:
962 printk(KERN_ERR "%s: unknown OPC 0x%02x\n",
963 __func__, dma_hw_desc->opc);
964 BUG();
965 }
966 case PPC440SPE_XOR_ID:
967 /* up to 16 sources */
968 xor_hw_desc = desc->hw_desc;
969 return xor_hw_desc->cbc & XOR_CDCR_OAC_MSK;
970 default:
971 BUG();
972 }
973 return 0;
974}
975
976/**
977 * ppc440spe_desc_get_dst_num - get the number of destination addresses in
978 * this descriptor
979 */
980static u32 ppc440spe_desc_get_dst_num(struct ppc440spe_adma_desc_slot *desc,
981 struct ppc440spe_adma_chan *chan)
982{
983 struct dma_cdb *dma_hw_desc;
984
985 switch (chan->device->id) {
986 case PPC440SPE_DMA0_ID:
987 case PPC440SPE_DMA1_ID:
988 /* May be 1 or 2 destinations */
989 dma_hw_desc = desc->hw_desc;
990 switch (dma_hw_desc->opc) {
991 case DMA_CDB_OPC_NO_OP:
992 case DMA_CDB_OPC_DCHECK128:
993 return 0;
994 case DMA_CDB_OPC_MV_SG1_SG2:
995 case DMA_CDB_OPC_DFILL128:
996 return 1;
997 case DMA_CDB_OPC_MULTICAST:
998 if (desc->dst_cnt == 2)
999 return 2;
1000 else
1001 return 1;
1002 default:
1003 printk(KERN_ERR "%s: unknown OPC 0x%02x\n",
1004 __func__, dma_hw_desc->opc);
1005 BUG();
1006 }
1007 case PPC440SPE_XOR_ID:
1008 /* Always only 1 destination */
1009 return 1;
1010 default:
1011 BUG();
1012 }
1013 return 0;
1014}
1015
1016/**
1017 * ppc440spe_desc_get_link - get the address of the descriptor that 805 * ppc440spe_desc_get_link - get the address of the descriptor that
1018 * follows this one 806 * follows this one
1019 */ 807 */
@@ -1705,43 +1493,6 @@ static void ppc440spe_adma_free_slots(struct ppc440spe_adma_desc_slot *slot,
1705 } 1493 }
1706} 1494}
1707 1495
1708static void ppc440spe_adma_unmap(struct ppc440spe_adma_chan *chan,
1709 struct ppc440spe_adma_desc_slot *desc)
1710{
1711 u32 src_cnt, dst_cnt;
1712 dma_addr_t addr;
1713
1714 /*
1715 * get the number of sources & destination
1716 * included in this descriptor and unmap
1717 * them all
1718 */
1719 src_cnt = ppc440spe_desc_get_src_num(desc, chan);
1720 dst_cnt = ppc440spe_desc_get_dst_num(desc, chan);
1721
1722 /* unmap destinations */
1723 if (!(desc->async_tx.flags & DMA_COMPL_SKIP_DEST_UNMAP)) {
1724 while (dst_cnt--) {
1725 addr = ppc440spe_desc_get_dest_addr(
1726 desc, chan, dst_cnt);
1727 dma_unmap_page(chan->device->dev,
1728 addr, desc->unmap_len,
1729 DMA_FROM_DEVICE);
1730 }
1731 }
1732
1733 /* unmap sources */
1734 if (!(desc->async_tx.flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
1735 while (src_cnt--) {
1736 addr = ppc440spe_desc_get_src_addr(
1737 desc, chan, src_cnt);
1738 dma_unmap_page(chan->device->dev,
1739 addr, desc->unmap_len,
1740 DMA_TO_DEVICE);
1741 }
1742 }
1743}
1744
1745/** 1496/**
1746 * ppc440spe_adma_run_tx_complete_actions - call functions to be called 1497 * ppc440spe_adma_run_tx_complete_actions - call functions to be called
1747 * upon completion 1498 * upon completion
@@ -1765,26 +1516,7 @@ static dma_cookie_t ppc440spe_adma_run_tx_complete_actions(
1765 desc->async_tx.callback( 1516 desc->async_tx.callback(
1766 desc->async_tx.callback_param); 1517 desc->async_tx.callback_param);
1767 1518
1768 /* unmap dma addresses 1519 dma_descriptor_unmap(&desc->async_tx);
1769 * (unmap_single vs unmap_page?)
1770 *
1771 * actually, ppc's dma_unmap_page() functions are empty, so
1772 * the following code is just for the sake of completeness
1773 */
1774 if (chan && chan->needs_unmap && desc->group_head &&
1775 desc->unmap_len) {
1776 struct ppc440spe_adma_desc_slot *unmap =
1777 desc->group_head;
1778 /* assume 1 slot per op always */
1779 u32 slot_count = unmap->slot_cnt;
1780
1781 /* Run through the group list and unmap addresses */
1782 for (i = 0; i < slot_count; i++) {
1783 BUG_ON(!unmap);
1784 ppc440spe_adma_unmap(chan, unmap);
1785 unmap = unmap->hw_next;
1786 }
1787 }
1788 } 1520 }
1789 1521
1790 /* run dependent operations */ 1522 /* run dependent operations */
diff --git a/drivers/dma/timb_dma.c b/drivers/dma/timb_dma.c
index 28af214fce04..4506a7b4f972 100644
--- a/drivers/dma/timb_dma.c
+++ b/drivers/dma/timb_dma.c
@@ -154,38 +154,6 @@ static bool __td_dma_done_ack(struct timb_dma_chan *td_chan)
154 return done; 154 return done;
155} 155}
156 156
157static void __td_unmap_desc(struct timb_dma_chan *td_chan, const u8 *dma_desc,
158 bool single)
159{
160 dma_addr_t addr;
161 int len;
162
163 addr = (dma_desc[7] << 24) | (dma_desc[6] << 16) | (dma_desc[5] << 8) |
164 dma_desc[4];
165
166 len = (dma_desc[3] << 8) | dma_desc[2];
167
168 if (single)
169 dma_unmap_single(chan2dev(&td_chan->chan), addr, len,
170 DMA_TO_DEVICE);
171 else
172 dma_unmap_page(chan2dev(&td_chan->chan), addr, len,
173 DMA_TO_DEVICE);
174}
175
176static void __td_unmap_descs(struct timb_dma_desc *td_desc, bool single)
177{
178 struct timb_dma_chan *td_chan = container_of(td_desc->txd.chan,
179 struct timb_dma_chan, chan);
180 u8 *descs;
181
182 for (descs = td_desc->desc_list; ; descs += TIMB_DMA_DESC_SIZE) {
183 __td_unmap_desc(td_chan, descs, single);
184 if (descs[0] & 0x02)
185 break;
186 }
187}
188
189static int td_fill_desc(struct timb_dma_chan *td_chan, u8 *dma_desc, 157static int td_fill_desc(struct timb_dma_chan *td_chan, u8 *dma_desc,
190 struct scatterlist *sg, bool last) 158 struct scatterlist *sg, bool last)
191{ 159{
@@ -293,10 +261,7 @@ static void __td_finish(struct timb_dma_chan *td_chan)
293 261
294 list_move(&td_desc->desc_node, &td_chan->free_list); 262 list_move(&td_desc->desc_node, &td_chan->free_list);
295 263
296 if (!(txd->flags & DMA_COMPL_SKIP_SRC_UNMAP)) 264 dma_descriptor_unmap(txd);
297 __td_unmap_descs(td_desc,
298 txd->flags & DMA_COMPL_SRC_UNMAP_SINGLE);
299
300 /* 265 /*
301 * The API requires that no submissions are done from a 266 * The API requires that no submissions are done from a
302 * callback, so we don't need to drop the lock here 267 * callback, so we don't need to drop the lock here
diff --git a/drivers/dma/txx9dmac.c b/drivers/dma/txx9dmac.c
index c2829b481bf2..bae6c29f5502 100644
--- a/drivers/dma/txx9dmac.c
+++ b/drivers/dma/txx9dmac.c
@@ -419,30 +419,7 @@ txx9dmac_descriptor_complete(struct txx9dmac_chan *dc,
419 list_splice_init(&desc->tx_list, &dc->free_list); 419 list_splice_init(&desc->tx_list, &dc->free_list);
420 list_move(&desc->desc_node, &dc->free_list); 420 list_move(&desc->desc_node, &dc->free_list);
421 421
422 if (!ds) { 422 dma_descriptor_unmap(txd);
423 dma_addr_t dmaaddr;
424 if (!(txd->flags & DMA_COMPL_SKIP_DEST_UNMAP)) {
425 dmaaddr = is_dmac64(dc) ?
426 desc->hwdesc.DAR : desc->hwdesc32.DAR;
427 if (txd->flags & DMA_COMPL_DEST_UNMAP_SINGLE)
428 dma_unmap_single(chan2parent(&dc->chan),
429 dmaaddr, desc->len, DMA_FROM_DEVICE);
430 else
431 dma_unmap_page(chan2parent(&dc->chan),
432 dmaaddr, desc->len, DMA_FROM_DEVICE);
433 }
434 if (!(txd->flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
435 dmaaddr = is_dmac64(dc) ?
436 desc->hwdesc.SAR : desc->hwdesc32.SAR;
437 if (txd->flags & DMA_COMPL_SRC_UNMAP_SINGLE)
438 dma_unmap_single(chan2parent(&dc->chan),
439 dmaaddr, desc->len, DMA_TO_DEVICE);
440 else
441 dma_unmap_page(chan2parent(&dc->chan),
442 dmaaddr, desc->len, DMA_TO_DEVICE);
443 }
444 }
445
446 /* 423 /*
447 * The API requires that no submissions are done from a 424 * The API requires that no submissions are done from a
448 * callback, so we don't need to drop the lock here 425 * callback, so we don't need to drop the lock here
generated by cgit v1.2.2 (git 2.25.0) at 2025-02-23 23:17:08 -0500