aboutsummaryrefslogtreecommitdiffstats
path: root/drivers
diff options
context:
space:
mode:
Diffstat (limited to 'drivers')
-rw-r--r--drivers/dma/Kconfig20
-rw-r--r--drivers/dma/Makefile2
-rw-r--r--drivers/dma/at_hdmac.c4
-rw-r--r--drivers/dma/coh901318.c169
-rw-r--r--drivers/dma/dmatest.c2
-rw-r--r--drivers/dma/intel_mid_dma.c1143
-rw-r--r--drivers/dma/intel_mid_dma_regs.h260
-rw-r--r--drivers/dma/ioat/dma.h1
-rw-r--r--drivers/dma/ioat/dma_v2.c24
-rw-r--r--drivers/dma/ioat/dma_v3.c5
-rw-r--r--drivers/dma/pch_dma.c957
-rw-r--r--drivers/dma/ste_dma40.c860
-rw-r--r--drivers/dma/ste_dma40_ll.c40
-rw-r--r--drivers/dma/ste_dma40_ll.h15
-rw-r--r--drivers/dma/timb_dma.c8
-rw-r--r--drivers/pci/intel-iommu.c28
16 files changed, 3174 insertions, 364 deletions
diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig
index 9e01e96fee94..fed57634b6c1 100644
--- a/drivers/dma/Kconfig
+++ b/drivers/dma/Kconfig
@@ -33,6 +33,19 @@ if DMADEVICES
33 33
34comment "DMA Devices" 34comment "DMA Devices"
35 35
36config INTEL_MID_DMAC
37 tristate "Intel MID DMA support for Peripheral DMA controllers"
38 depends on PCI && X86
39 select DMA_ENGINE
40 default n
41 help
42 Enable support for the Intel(R) MID DMA engine present
43 in Intel MID chipsets.
44
45 Say Y here if you have such a chipset.
46
47 If unsure, say N.
48
36config ASYNC_TX_DISABLE_CHANNEL_SWITCH 49config ASYNC_TX_DISABLE_CHANNEL_SWITCH
37 bool 50 bool
38 51
@@ -175,6 +188,13 @@ config PL330_DMA
175 You need to provide platform specific settings via 188 You need to provide platform specific settings via
176 platform_data for a dma-pl330 device. 189 platform_data for a dma-pl330 device.
177 190
191config PCH_DMA
192 tristate "Topcliff PCH DMA support"
193 depends on PCI && X86
194 select DMA_ENGINE
195 help
196 Enable support for the Topcliff PCH DMA engine.
197
178config DMA_ENGINE 198config DMA_ENGINE
179 bool 199 bool
180 200
diff --git a/drivers/dma/Makefile b/drivers/dma/Makefile
index 0fe5ebbfda5d..72bd70384d8a 100644
--- a/drivers/dma/Makefile
+++ b/drivers/dma/Makefile
@@ -7,6 +7,7 @@ endif
7 7
8obj-$(CONFIG_DMA_ENGINE) += dmaengine.o 8obj-$(CONFIG_DMA_ENGINE) += dmaengine.o
9obj-$(CONFIG_NET_DMA) += iovlock.o 9obj-$(CONFIG_NET_DMA) += iovlock.o
10obj-$(CONFIG_INTEL_MID_DMAC) += intel_mid_dma.o
10obj-$(CONFIG_DMATEST) += dmatest.o 11obj-$(CONFIG_DMATEST) += dmatest.o
11obj-$(CONFIG_INTEL_IOATDMA) += ioat/ 12obj-$(CONFIG_INTEL_IOATDMA) += ioat/
12obj-$(CONFIG_INTEL_IOP_ADMA) += iop-adma.o 13obj-$(CONFIG_INTEL_IOP_ADMA) += iop-adma.o
@@ -23,3 +24,4 @@ obj-$(CONFIG_AMCC_PPC440SPE_ADMA) += ppc4xx/
23obj-$(CONFIG_TIMB_DMA) += timb_dma.o 24obj-$(CONFIG_TIMB_DMA) += timb_dma.o
24obj-$(CONFIG_STE_DMA40) += ste_dma40.o ste_dma40_ll.o 25obj-$(CONFIG_STE_DMA40) += ste_dma40.o ste_dma40_ll.o
25obj-$(CONFIG_PL330_DMA) += pl330.o 26obj-$(CONFIG_PL330_DMA) += pl330.o
27obj-$(CONFIG_PCH_DMA) += pch_dma.o
diff --git a/drivers/dma/at_hdmac.c b/drivers/dma/at_hdmac.c
index e88076022a7a..a0f3e6a06e06 100644
--- a/drivers/dma/at_hdmac.c
+++ b/drivers/dma/at_hdmac.c
@@ -790,12 +790,12 @@ static int atc_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd,
790 list_splice_init(&atchan->queue, &list); 790 list_splice_init(&atchan->queue, &list);
791 list_splice_init(&atchan->active_list, &list); 791 list_splice_init(&atchan->active_list, &list);
792 792
793 spin_unlock_bh(&atchan->lock);
794
795 /* Flush all pending and queued descriptors */ 793 /* Flush all pending and queued descriptors */
796 list_for_each_entry_safe(desc, _desc, &list, desc_node) 794 list_for_each_entry_safe(desc, _desc, &list, desc_node)
797 atc_chain_complete(atchan, desc); 795 atc_chain_complete(atchan, desc);
798 796
797 spin_unlock_bh(&atchan->lock);
798
799 return 0; 799 return 0;
800} 800}
801 801
diff --git a/drivers/dma/coh901318.c b/drivers/dma/coh901318.c
index a724e6be1b4d..557e2272e5b3 100644
--- a/drivers/dma/coh901318.c
+++ b/drivers/dma/coh901318.c
@@ -72,6 +72,9 @@ struct coh901318_chan {
72 unsigned long nbr_active_done; 72 unsigned long nbr_active_done;
73 unsigned long busy; 73 unsigned long busy;
74 74
75 u32 runtime_addr;
76 u32 runtime_ctrl;
77
75 struct coh901318_base *base; 78 struct coh901318_base *base;
76}; 79};
77 80
@@ -190,6 +193,9 @@ static inline struct coh901318_chan *to_coh901318_chan(struct dma_chan *chan)
190static inline dma_addr_t 193static inline dma_addr_t
191cohc_dev_addr(struct coh901318_chan *cohc) 194cohc_dev_addr(struct coh901318_chan *cohc)
192{ 195{
196 /* Runtime supplied address will take precedence */
197 if (cohc->runtime_addr)
198 return cohc->runtime_addr;
193 return cohc->base->platform->chan_conf[cohc->id].dev_addr; 199 return cohc->base->platform->chan_conf[cohc->id].dev_addr;
194} 200}
195 201
@@ -1055,6 +1061,14 @@ coh901318_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
1055 1061
1056 params = cohc_chan_param(cohc); 1062 params = cohc_chan_param(cohc);
1057 config = params->config; 1063 config = params->config;
1064 /*
1065 * Add runtime-specific control on top, make
1066 * sure the bits you set per peripheral channel are
1067 * cleared in the default config from the platform.
1068 */
1069 ctrl_chained |= cohc->runtime_ctrl;
1070 ctrl_last |= cohc->runtime_ctrl;
1071 ctrl |= cohc->runtime_ctrl;
1058 1072
1059 if (direction == DMA_TO_DEVICE) { 1073 if (direction == DMA_TO_DEVICE) {
1060 u32 tx_flags = COH901318_CX_CTRL_PRDD_SOURCE | 1074 u32 tx_flags = COH901318_CX_CTRL_PRDD_SOURCE |
@@ -1113,6 +1127,12 @@ coh901318_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
1113 if (ret) 1127 if (ret)
1114 goto err_lli_fill; 1128 goto err_lli_fill;
1115 1129
1130 /*
1131 * Set the default ctrl for the channel to the one from the lli,
1132 * things may have changed due to odd buffer alignment etc.
1133 */
1134 coh901318_set_ctrl(cohc, lli->control);
1135
1116 COH_DBG(coh901318_list_print(cohc, lli)); 1136 COH_DBG(coh901318_list_print(cohc, lli));
1117 1137
1118 /* Pick a descriptor to handle this transfer */ 1138 /* Pick a descriptor to handle this transfer */
@@ -1175,6 +1195,146 @@ coh901318_issue_pending(struct dma_chan *chan)
1175 spin_unlock_irqrestore(&cohc->lock, flags); 1195 spin_unlock_irqrestore(&cohc->lock, flags);
1176} 1196}
1177 1197
1198/*
1199 * Here we wrap in the runtime dma control interface
1200 */
1201struct burst_table {
1202 int burst_8bit;
1203 int burst_16bit;
1204 int burst_32bit;
1205 u32 reg;
1206};
1207
1208static const struct burst_table burst_sizes[] = {
1209 {
1210 .burst_8bit = 64,
1211 .burst_16bit = 32,
1212 .burst_32bit = 16,
1213 .reg = COH901318_CX_CTRL_BURST_COUNT_64_BYTES,
1214 },
1215 {
1216 .burst_8bit = 48,
1217 .burst_16bit = 24,
1218 .burst_32bit = 12,
1219 .reg = COH901318_CX_CTRL_BURST_COUNT_48_BYTES,
1220 },
1221 {
1222 .burst_8bit = 32,
1223 .burst_16bit = 16,
1224 .burst_32bit = 8,
1225 .reg = COH901318_CX_CTRL_BURST_COUNT_32_BYTES,
1226 },
1227 {
1228 .burst_8bit = 16,
1229 .burst_16bit = 8,
1230 .burst_32bit = 4,
1231 .reg = COH901318_CX_CTRL_BURST_COUNT_16_BYTES,
1232 },
1233 {
1234 .burst_8bit = 8,
1235 .burst_16bit = 4,
1236 .burst_32bit = 2,
1237 .reg = COH901318_CX_CTRL_BURST_COUNT_8_BYTES,
1238 },
1239 {
1240 .burst_8bit = 4,
1241 .burst_16bit = 2,
1242 .burst_32bit = 1,
1243 .reg = COH901318_CX_CTRL_BURST_COUNT_4_BYTES,
1244 },
1245 {
1246 .burst_8bit = 2,
1247 .burst_16bit = 1,
1248 .burst_32bit = 0,
1249 .reg = COH901318_CX_CTRL_BURST_COUNT_2_BYTES,
1250 },
1251 {
1252 .burst_8bit = 1,
1253 .burst_16bit = 0,
1254 .burst_32bit = 0,
1255 .reg = COH901318_CX_CTRL_BURST_COUNT_1_BYTE,
1256 },
1257};
1258
1259static void coh901318_dma_set_runtimeconfig(struct dma_chan *chan,
1260 struct dma_slave_config *config)
1261{
1262 struct coh901318_chan *cohc = to_coh901318_chan(chan);
1263 dma_addr_t addr;
1264 enum dma_slave_buswidth addr_width;
1265 u32 maxburst;
1266 u32 runtime_ctrl = 0;
1267 int i = 0;
1268
1269 /* We only support mem to per or per to mem transfers */
1270 if (config->direction == DMA_FROM_DEVICE) {
1271 addr = config->src_addr;
1272 addr_width = config->src_addr_width;
1273 maxburst = config->src_maxburst;
1274 } else if (config->direction == DMA_TO_DEVICE) {
1275 addr = config->dst_addr;
1276 addr_width = config->dst_addr_width;
1277 maxburst = config->dst_maxburst;
1278 } else {
1279 dev_err(COHC_2_DEV(cohc), "illegal channel mode\n");
1280 return;
1281 }
1282
1283 dev_dbg(COHC_2_DEV(cohc), "configure channel for %d byte transfers\n",
1284 addr_width);
1285 switch (addr_width) {
1286 case DMA_SLAVE_BUSWIDTH_1_BYTE:
1287 runtime_ctrl |=
1288 COH901318_CX_CTRL_SRC_BUS_SIZE_8_BITS |
1289 COH901318_CX_CTRL_DST_BUS_SIZE_8_BITS;
1290
1291 while (i < ARRAY_SIZE(burst_sizes)) {
1292 if (burst_sizes[i].burst_8bit <= maxburst)
1293 break;
1294 i++;
1295 }
1296
1297 break;
1298 case DMA_SLAVE_BUSWIDTH_2_BYTES:
1299 runtime_ctrl |=
1300 COH901318_CX_CTRL_SRC_BUS_SIZE_16_BITS |
1301 COH901318_CX_CTRL_DST_BUS_SIZE_16_BITS;
1302
1303 while (i < ARRAY_SIZE(burst_sizes)) {
1304 if (burst_sizes[i].burst_16bit <= maxburst)
1305 break;
1306 i++;
1307 }
1308
1309 break;
1310 case DMA_SLAVE_BUSWIDTH_4_BYTES:
1311 /* Direction doesn't matter here, it's 32/32 bits */
1312 runtime_ctrl |=
1313 COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS |
1314 COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS;
1315
1316 while (i < ARRAY_SIZE(burst_sizes)) {
1317 if (burst_sizes[i].burst_32bit <= maxburst)
1318 break;
1319 i++;
1320 }
1321
1322 break;
1323 default:
1324 dev_err(COHC_2_DEV(cohc),
1325 "bad runtimeconfig: alien address width\n");
1326 return;
1327 }
1328
1329 runtime_ctrl |= burst_sizes[i].reg;
1330 dev_dbg(COHC_2_DEV(cohc),
1331 "selected burst size %d bytes for address width %d bytes, maxburst %d\n",
1332 burst_sizes[i].burst_8bit, addr_width, maxburst);
1333
1334 cohc->runtime_addr = addr;
1335 cohc->runtime_ctrl = runtime_ctrl;
1336}
1337
1178static int 1338static int
1179coh901318_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd, 1339coh901318_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd,
1180 unsigned long arg) 1340 unsigned long arg)
@@ -1184,6 +1344,14 @@ coh901318_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd,
1184 struct coh901318_desc *cohd; 1344 struct coh901318_desc *cohd;
1185 void __iomem *virtbase = cohc->base->virtbase; 1345 void __iomem *virtbase = cohc->base->virtbase;
1186 1346
1347 if (cmd == DMA_SLAVE_CONFIG) {
1348 struct dma_slave_config *config =
1349 (struct dma_slave_config *) arg;
1350
1351 coh901318_dma_set_runtimeconfig(chan, config);
1352 return 0;
1353 }
1354
1187 if (cmd == DMA_PAUSE) { 1355 if (cmd == DMA_PAUSE) {
1188 coh901318_pause(chan); 1356 coh901318_pause(chan);
1189 return 0; 1357 return 0;
@@ -1240,6 +1408,7 @@ coh901318_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd,
1240 1408
1241 return 0; 1409 return 0;
1242} 1410}
1411
1243void coh901318_base_init(struct dma_device *dma, const int *pick_chans, 1412void coh901318_base_init(struct dma_device *dma, const int *pick_chans,
1244 struct coh901318_base *base) 1413 struct coh901318_base *base)
1245{ 1414{
diff --git a/drivers/dma/dmatest.c b/drivers/dma/dmatest.c
index 68d58c414cf0..5589358b684d 100644
--- a/drivers/dma/dmatest.c
+++ b/drivers/dma/dmatest.c
@@ -540,7 +540,7 @@ static int dmatest_add_channel(struct dma_chan *chan)
540 struct dmatest_chan *dtc; 540 struct dmatest_chan *dtc;
541 struct dma_device *dma_dev = chan->device; 541 struct dma_device *dma_dev = chan->device;
542 unsigned int thread_count = 0; 542 unsigned int thread_count = 0;
543 unsigned int cnt; 543 int cnt;
544 544
545 dtc = kmalloc(sizeof(struct dmatest_chan), GFP_KERNEL); 545 dtc = kmalloc(sizeof(struct dmatest_chan), GFP_KERNEL);
546 if (!dtc) { 546 if (!dtc) {
diff --git a/drivers/dma/intel_mid_dma.c b/drivers/dma/intel_mid_dma.c
new file mode 100644
index 000000000000..c2591e8d9b6e
--- /dev/null
+++ b/drivers/dma/intel_mid_dma.c
@@ -0,0 +1,1143 @@
1/*
2 * intel_mid_dma.c - Intel Langwell DMA Drivers
3 *
4 * Copyright (C) 2008-10 Intel Corp
5 * Author: Vinod Koul <vinod.koul@intel.com>
6 * The driver design is based on dw_dmac driver
7 * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
8 *
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License as published by
11 * the Free Software Foundation; version 2 of the License.
12 *
13 * This program is distributed in the hope that it will be useful, but
14 * WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 * General Public License for more details.
17 *
18 * You should have received a copy of the GNU General Public License along
19 * with this program; if not, write to the Free Software Foundation, Inc.,
20 * 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA.
21 *
22 * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
23 *
24 *
25 */
26#include <linux/pci.h>
27#include <linux/interrupt.h>
28#include <linux/intel_mid_dma.h>
29
30#define MAX_CHAN 4 /*max ch across controllers*/
31#include "intel_mid_dma_regs.h"
32
33#define INTEL_MID_DMAC1_ID 0x0814
34#define INTEL_MID_DMAC2_ID 0x0813
35#define INTEL_MID_GP_DMAC2_ID 0x0827
36#define INTEL_MFLD_DMAC1_ID 0x0830
37#define LNW_PERIPHRAL_MASK_BASE 0xFFAE8008
38#define LNW_PERIPHRAL_MASK_SIZE 0x10
39#define LNW_PERIPHRAL_STATUS 0x0
40#define LNW_PERIPHRAL_MASK 0x8
41
42struct intel_mid_dma_probe_info {
43 u8 max_chan;
44 u8 ch_base;
45 u16 block_size;
46 u32 pimr_mask;
47};
48
49#define INFO(_max_chan, _ch_base, _block_size, _pimr_mask) \
50 ((kernel_ulong_t)&(struct intel_mid_dma_probe_info) { \
51 .max_chan = (_max_chan), \
52 .ch_base = (_ch_base), \
53 .block_size = (_block_size), \
54 .pimr_mask = (_pimr_mask), \
55 })
56
57/*****************************************************************************
58Utility Functions*/
59/**
60 * get_ch_index - convert status to channel
61 * @status: status mask
62 * @base: dma ch base value
63 *
64 * Modify the status mask and return the channel index needing
65 * attention (or -1 if neither)
66 */
67static int get_ch_index(int *status, unsigned int base)
68{
69 int i;
70 for (i = 0; i < MAX_CHAN; i++) {
71 if (*status & (1 << (i + base))) {
72 *status = *status & ~(1 << (i + base));
73 pr_debug("MDMA: index %d New status %x\n", i, *status);
74 return i;
75 }
76 }
77 return -1;
78}
79
80/**
81 * get_block_ts - calculates dma transaction length
82 * @len: dma transfer length
83 * @tx_width: dma transfer src width
84 * @block_size: dma controller max block size
85 *
86 * Based on src width calculate the DMA trsaction length in data items
87 * return data items or FFFF if exceeds max length for block
88 */
89static int get_block_ts(int len, int tx_width, int block_size)
90{
91 int byte_width = 0, block_ts = 0;
92
93 switch (tx_width) {
94 case LNW_DMA_WIDTH_8BIT:
95 byte_width = 1;
96 break;
97 case LNW_DMA_WIDTH_16BIT:
98 byte_width = 2;
99 break;
100 case LNW_DMA_WIDTH_32BIT:
101 default:
102 byte_width = 4;
103 break;
104 }
105
106 block_ts = len/byte_width;
107 if (block_ts > block_size)
108 block_ts = 0xFFFF;
109 return block_ts;
110}
111
112/*****************************************************************************
113DMAC1 interrupt Functions*/
114
115/**
116 * dmac1_mask_periphral_intr - mask the periphral interrupt
117 * @midc: dma channel for which masking is required
118 *
119 * Masks the DMA periphral interrupt
120 * this is valid for DMAC1 family controllers only
121 * This controller should have periphral mask registers already mapped
122 */
123static void dmac1_mask_periphral_intr(struct intel_mid_dma_chan *midc)
124{
125 u32 pimr;
126 struct middma_device *mid = to_middma_device(midc->chan.device);
127
128 if (mid->pimr_mask) {
129 pimr = readl(mid->mask_reg + LNW_PERIPHRAL_MASK);
130 pimr |= mid->pimr_mask;
131 writel(pimr, mid->mask_reg + LNW_PERIPHRAL_MASK);
132 }
133 return;
134}
135
136/**
137 * dmac1_unmask_periphral_intr - unmask the periphral interrupt
138 * @midc: dma channel for which masking is required
139 *
140 * UnMasks the DMA periphral interrupt,
141 * this is valid for DMAC1 family controllers only
142 * This controller should have periphral mask registers already mapped
143 */
144static void dmac1_unmask_periphral_intr(struct intel_mid_dma_chan *midc)
145{
146 u32 pimr;
147 struct middma_device *mid = to_middma_device(midc->chan.device);
148
149 if (mid->pimr_mask) {
150 pimr = readl(mid->mask_reg + LNW_PERIPHRAL_MASK);
151 pimr &= ~mid->pimr_mask;
152 writel(pimr, mid->mask_reg + LNW_PERIPHRAL_MASK);
153 }
154 return;
155}
156
157/**
158 * enable_dma_interrupt - enable the periphral interrupt
159 * @midc: dma channel for which enable interrupt is required
160 *
161 * Enable the DMA periphral interrupt,
162 * this is valid for DMAC1 family controllers only
163 * This controller should have periphral mask registers already mapped
164 */
165static void enable_dma_interrupt(struct intel_mid_dma_chan *midc)
166{
167 dmac1_unmask_periphral_intr(midc);
168
169 /*en ch interrupts*/
170 iowrite32(UNMASK_INTR_REG(midc->ch_id), midc->dma_base + MASK_TFR);
171 iowrite32(UNMASK_INTR_REG(midc->ch_id), midc->dma_base + MASK_ERR);
172 return;
173}
174
175/**
176 * disable_dma_interrupt - disable the periphral interrupt
177 * @midc: dma channel for which disable interrupt is required
178 *
179 * Disable the DMA periphral interrupt,
180 * this is valid for DMAC1 family controllers only
181 * This controller should have periphral mask registers already mapped
182 */
183static void disable_dma_interrupt(struct intel_mid_dma_chan *midc)
184{
185 /*Check LPE PISR, make sure fwd is disabled*/
186 dmac1_mask_periphral_intr(midc);
187 iowrite32(MASK_INTR_REG(midc->ch_id), midc->dma_base + MASK_BLOCK);
188 iowrite32(MASK_INTR_REG(midc->ch_id), midc->dma_base + MASK_TFR);
189 iowrite32(MASK_INTR_REG(midc->ch_id), midc->dma_base + MASK_ERR);
190 return;
191}
192
193/*****************************************************************************
194DMA channel helper Functions*/
195/**
196 * mid_desc_get - get a descriptor
197 * @midc: dma channel for which descriptor is required
198 *
199 * Obtain a descriptor for the channel. Returns NULL if none are free.
200 * Once the descriptor is returned it is private until put on another
201 * list or freed
202 */
203static struct intel_mid_dma_desc *midc_desc_get(struct intel_mid_dma_chan *midc)
204{
205 struct intel_mid_dma_desc *desc, *_desc;
206 struct intel_mid_dma_desc *ret = NULL;
207
208 spin_lock_bh(&midc->lock);
209 list_for_each_entry_safe(desc, _desc, &midc->free_list, desc_node) {
210 if (async_tx_test_ack(&desc->txd)) {
211 list_del(&desc->desc_node);
212 ret = desc;
213 break;
214 }
215 }
216 spin_unlock_bh(&midc->lock);
217 return ret;
218}
219
220/**
221 * mid_desc_put - put a descriptor
222 * @midc: dma channel for which descriptor is required
223 * @desc: descriptor to put
224 *
225 * Return a descriptor from lwn_desc_get back to the free pool
226 */
227static void midc_desc_put(struct intel_mid_dma_chan *midc,
228 struct intel_mid_dma_desc *desc)
229{
230 if (desc) {
231 spin_lock_bh(&midc->lock);
232 list_add_tail(&desc->desc_node, &midc->free_list);
233 spin_unlock_bh(&midc->lock);
234 }
235}
236/**
237 * midc_dostart - begin a DMA transaction
238 * @midc: channel for which txn is to be started
239 * @first: first descriptor of series
240 *
241 * Load a transaction into the engine. This must be called with midc->lock
242 * held and bh disabled.
243 */
244static void midc_dostart(struct intel_mid_dma_chan *midc,
245 struct intel_mid_dma_desc *first)
246{
247 struct middma_device *mid = to_middma_device(midc->chan.device);
248
249 /* channel is idle */
250 if (midc->in_use && test_ch_en(midc->dma_base, midc->ch_id)) {
251 /*error*/
252 pr_err("ERR_MDMA: channel is busy in start\n");
253 /* The tasklet will hopefully advance the queue... */
254 return;
255 }
256
257 /*write registers and en*/
258 iowrite32(first->sar, midc->ch_regs + SAR);
259 iowrite32(first->dar, midc->ch_regs + DAR);
260 iowrite32(first->cfg_hi, midc->ch_regs + CFG_HIGH);
261 iowrite32(first->cfg_lo, midc->ch_regs + CFG_LOW);
262 iowrite32(first->ctl_lo, midc->ch_regs + CTL_LOW);
263 iowrite32(first->ctl_hi, midc->ch_regs + CTL_HIGH);
264 pr_debug("MDMA:TX SAR %x,DAR %x,CFGL %x,CFGH %x,CTLH %x, CTLL %x\n",
265 (int)first->sar, (int)first->dar, first->cfg_hi,
266 first->cfg_lo, first->ctl_hi, first->ctl_lo);
267
268 iowrite32(ENABLE_CHANNEL(midc->ch_id), mid->dma_base + DMA_CHAN_EN);
269 first->status = DMA_IN_PROGRESS;
270}
271
272/**
273 * midc_descriptor_complete - process completed descriptor
274 * @midc: channel owning the descriptor
275 * @desc: the descriptor itself
276 *
277 * Process a completed descriptor and perform any callbacks upon
278 * the completion. The completion handling drops the lock during the
279 * callbacks but must be called with the lock held.
280 */
281static void midc_descriptor_complete(struct intel_mid_dma_chan *midc,
282 struct intel_mid_dma_desc *desc)
283{
284 struct dma_async_tx_descriptor *txd = &desc->txd;
285 dma_async_tx_callback callback_txd = NULL;
286 void *param_txd = NULL;
287
288 midc->completed = txd->cookie;
289 callback_txd = txd->callback;
290 param_txd = txd->callback_param;
291
292 list_move(&desc->desc_node, &midc->free_list);
293
294 spin_unlock_bh(&midc->lock);
295 if (callback_txd) {
296 pr_debug("MDMA: TXD callback set ... calling\n");
297 callback_txd(param_txd);
298 spin_lock_bh(&midc->lock);
299 return;
300 }
301 spin_lock_bh(&midc->lock);
302
303}
304/**
305 * midc_scan_descriptors - check the descriptors in channel
306 * mark completed when tx is completete
307 * @mid: device
308 * @midc: channel to scan
309 *
310 * Walk the descriptor chain for the device and process any entries
311 * that are complete.
312 */
313static void midc_scan_descriptors(struct middma_device *mid,
314 struct intel_mid_dma_chan *midc)
315{
316 struct intel_mid_dma_desc *desc = NULL, *_desc = NULL;
317
318 /*tx is complete*/
319 list_for_each_entry_safe(desc, _desc, &midc->active_list, desc_node) {
320 if (desc->status == DMA_IN_PROGRESS) {
321 desc->status = DMA_SUCCESS;
322 midc_descriptor_complete(midc, desc);
323 }
324 }
325 return;
326}
327
328/*****************************************************************************
329DMA engine callback Functions*/
330/**
331 * intel_mid_dma_tx_submit - callback to submit DMA transaction
332 * @tx: dma engine descriptor
333 *
334 * Submit the DMA trasaction for this descriptor, start if ch idle
335 */
336static dma_cookie_t intel_mid_dma_tx_submit(struct dma_async_tx_descriptor *tx)
337{
338 struct intel_mid_dma_desc *desc = to_intel_mid_dma_desc(tx);
339 struct intel_mid_dma_chan *midc = to_intel_mid_dma_chan(tx->chan);
340 dma_cookie_t cookie;
341
342 spin_lock_bh(&midc->lock);
343 cookie = midc->chan.cookie;
344
345 if (++cookie < 0)
346 cookie = 1;
347
348 midc->chan.cookie = cookie;
349 desc->txd.cookie = cookie;
350
351
352 if (list_empty(&midc->active_list)) {
353 midc_dostart(midc, desc);
354 list_add_tail(&desc->desc_node, &midc->active_list);
355 } else {
356 list_add_tail(&desc->desc_node, &midc->queue);
357 }
358 spin_unlock_bh(&midc->lock);
359
360 return cookie;
361}
362
363/**
364 * intel_mid_dma_issue_pending - callback to issue pending txn
365 * @chan: chan where pending trascation needs to be checked and submitted
366 *
367 * Call for scan to issue pending descriptors
368 */
369static void intel_mid_dma_issue_pending(struct dma_chan *chan)
370{
371 struct intel_mid_dma_chan *midc = to_intel_mid_dma_chan(chan);
372
373 spin_lock_bh(&midc->lock);
374 if (!list_empty(&midc->queue))
375 midc_scan_descriptors(to_middma_device(chan->device), midc);
376 spin_unlock_bh(&midc->lock);
377}
378
379/**
380 * intel_mid_dma_tx_status - Return status of txn
381 * @chan: chan for where status needs to be checked
382 * @cookie: cookie for txn
383 * @txstate: DMA txn state
384 *
385 * Return status of DMA txn
386 */
387static enum dma_status intel_mid_dma_tx_status(struct dma_chan *chan,
388 dma_cookie_t cookie,
389 struct dma_tx_state *txstate)
390{
391 struct intel_mid_dma_chan *midc = to_intel_mid_dma_chan(chan);
392 dma_cookie_t last_used;
393 dma_cookie_t last_complete;
394 int ret;
395
396 last_complete = midc->completed;
397 last_used = chan->cookie;
398
399 ret = dma_async_is_complete(cookie, last_complete, last_used);
400 if (ret != DMA_SUCCESS) {
401 midc_scan_descriptors(to_middma_device(chan->device), midc);
402
403 last_complete = midc->completed;
404 last_used = chan->cookie;
405
406 ret = dma_async_is_complete(cookie, last_complete, last_used);
407 }
408
409 if (txstate) {
410 txstate->last = last_complete;
411 txstate->used = last_used;
412 txstate->residue = 0;
413 }
414 return ret;
415}
416
417/**
418 * intel_mid_dma_device_control - DMA device control
419 * @chan: chan for DMA control
420 * @cmd: control cmd
421 * @arg: cmd arg value
422 *
423 * Perform DMA control command
424 */
425static int intel_mid_dma_device_control(struct dma_chan *chan,
426 enum dma_ctrl_cmd cmd, unsigned long arg)
427{
428 struct intel_mid_dma_chan *midc = to_intel_mid_dma_chan(chan);
429 struct middma_device *mid = to_middma_device(chan->device);
430 struct intel_mid_dma_desc *desc, *_desc;
431 LIST_HEAD(list);
432
433 if (cmd != DMA_TERMINATE_ALL)
434 return -ENXIO;
435
436 spin_lock_bh(&midc->lock);
437 if (midc->in_use == false) {
438 spin_unlock_bh(&midc->lock);
439 return 0;
440 }
441 list_splice_init(&midc->free_list, &list);
442 midc->descs_allocated = 0;
443 midc->slave = NULL;
444
445 /* Disable interrupts */
446 disable_dma_interrupt(midc);
447
448 spin_unlock_bh(&midc->lock);
449 list_for_each_entry_safe(desc, _desc, &list, desc_node) {
450 pr_debug("MDMA: freeing descriptor %p\n", desc);
451 pci_pool_free(mid->dma_pool, desc, desc->txd.phys);
452 }
453 return 0;
454}
455
456/**
457 * intel_mid_dma_prep_slave_sg - Prep slave sg txn
458 * @chan: chan for DMA transfer
459 * @sgl: scatter gather list
460 * @sg_len: length of sg txn
461 * @direction: DMA transfer dirtn
462 * @flags: DMA flags
463 *
464 * Do DMA sg txn: NOT supported now
465 */
466static struct dma_async_tx_descriptor *intel_mid_dma_prep_slave_sg(
467 struct dma_chan *chan, struct scatterlist *sgl,
468 unsigned int sg_len, enum dma_data_direction direction,
469 unsigned long flags)
470{
471 /*not supported now*/
472 return NULL;
473}
474
475/**
476 * intel_mid_dma_prep_memcpy - Prep memcpy txn
477 * @chan: chan for DMA transfer
478 * @dest: destn address
479 * @src: src address
480 * @len: DMA transfer len
481 * @flags: DMA flags
482 *
483 * Perform a DMA memcpy. Note we support slave periphral DMA transfers only
484 * The periphral txn details should be filled in slave structure properly
485 * Returns the descriptor for this txn
486 */
487static struct dma_async_tx_descriptor *intel_mid_dma_prep_memcpy(
488 struct dma_chan *chan, dma_addr_t dest,
489 dma_addr_t src, size_t len, unsigned long flags)
490{
491 struct intel_mid_dma_chan *midc;
492 struct intel_mid_dma_desc *desc = NULL;
493 struct intel_mid_dma_slave *mids;
494 union intel_mid_dma_ctl_lo ctl_lo;
495 union intel_mid_dma_ctl_hi ctl_hi;
496 union intel_mid_dma_cfg_lo cfg_lo;
497 union intel_mid_dma_cfg_hi cfg_hi;
498 enum intel_mid_dma_width width = 0;
499
500 pr_debug("MDMA: Prep for memcpy\n");
501 WARN_ON(!chan);
502 if (!len)
503 return NULL;
504
505 mids = chan->private;
506 WARN_ON(!mids);
507
508 midc = to_intel_mid_dma_chan(chan);
509 WARN_ON(!midc);
510
511 pr_debug("MDMA:called for DMA %x CH %d Length %zu\n",
512 midc->dma->pci_id, midc->ch_id, len);
513 pr_debug("MDMA:Cfg passed Mode %x, Dirn %x, HS %x, Width %x\n",
514 mids->cfg_mode, mids->dirn, mids->hs_mode, mids->src_width);
515
516 /*calculate CFG_LO*/
517 if (mids->hs_mode == LNW_DMA_SW_HS) {
518 cfg_lo.cfg_lo = 0;
519 cfg_lo.cfgx.hs_sel_dst = 1;
520 cfg_lo.cfgx.hs_sel_src = 1;
521 } else if (mids->hs_mode == LNW_DMA_HW_HS)
522 cfg_lo.cfg_lo = 0x00000;
523
524 /*calculate CFG_HI*/
525 if (mids->cfg_mode == LNW_DMA_MEM_TO_MEM) {
526 /*SW HS only*/
527 cfg_hi.cfg_hi = 0;
528 } else {
529 cfg_hi.cfg_hi = 0;
530 if (midc->dma->pimr_mask) {
531 cfg_hi.cfgx.protctl = 0x0; /*default value*/
532 cfg_hi.cfgx.fifo_mode = 1;
533 if (mids->dirn == DMA_TO_DEVICE) {
534 cfg_hi.cfgx.src_per = 0;
535 if (mids->device_instance == 0)
536 cfg_hi.cfgx.dst_per = 3;
537 if (mids->device_instance == 1)
538 cfg_hi.cfgx.dst_per = 1;
539 } else if (mids->dirn == DMA_FROM_DEVICE) {
540 if (mids->device_instance == 0)
541 cfg_hi.cfgx.src_per = 2;
542 if (mids->device_instance == 1)
543 cfg_hi.cfgx.src_per = 0;
544 cfg_hi.cfgx.dst_per = 0;
545 }
546 } else {
547 cfg_hi.cfgx.protctl = 0x1; /*default value*/
548 cfg_hi.cfgx.src_per = cfg_hi.cfgx.dst_per =
549 midc->ch_id - midc->dma->chan_base;
550 }
551 }
552
553 /*calculate CTL_HI*/
554 ctl_hi.ctlx.reser = 0;
555 width = mids->src_width;
556
557 ctl_hi.ctlx.block_ts = get_block_ts(len, width, midc->dma->block_size);
558 pr_debug("MDMA:calc len %d for block size %d\n",
559 ctl_hi.ctlx.block_ts, midc->dma->block_size);
560 /*calculate CTL_LO*/
561 ctl_lo.ctl_lo = 0;
562 ctl_lo.ctlx.int_en = 1;
563 ctl_lo.ctlx.dst_tr_width = mids->dst_width;
564 ctl_lo.ctlx.src_tr_width = mids->src_width;
565 ctl_lo.ctlx.dst_msize = mids->src_msize;
566 ctl_lo.ctlx.src_msize = mids->dst_msize;
567
568 if (mids->cfg_mode == LNW_DMA_MEM_TO_MEM) {
569 ctl_lo.ctlx.tt_fc = 0;
570 ctl_lo.ctlx.sinc = 0;
571 ctl_lo.ctlx.dinc = 0;
572 } else {
573 if (mids->dirn == DMA_TO_DEVICE) {
574 ctl_lo.ctlx.sinc = 0;
575 ctl_lo.ctlx.dinc = 2;
576 ctl_lo.ctlx.tt_fc = 1;
577 } else if (mids->dirn == DMA_FROM_DEVICE) {
578 ctl_lo.ctlx.sinc = 2;
579 ctl_lo.ctlx.dinc = 0;
580 ctl_lo.ctlx.tt_fc = 2;
581 }
582 }
583
584 pr_debug("MDMA:Calc CTL LO %x, CTL HI %x, CFG LO %x, CFG HI %x\n",
585 ctl_lo.ctl_lo, ctl_hi.ctl_hi, cfg_lo.cfg_lo, cfg_hi.cfg_hi);
586
587 enable_dma_interrupt(midc);
588
589 desc = midc_desc_get(midc);
590 if (desc == NULL)
591 goto err_desc_get;
592 desc->sar = src;
593 desc->dar = dest ;
594 desc->len = len;
595 desc->cfg_hi = cfg_hi.cfg_hi;
596 desc->cfg_lo = cfg_lo.cfg_lo;
597 desc->ctl_lo = ctl_lo.ctl_lo;
598 desc->ctl_hi = ctl_hi.ctl_hi;
599 desc->width = width;
600 desc->dirn = mids->dirn;
601 return &desc->txd;
602
603err_desc_get:
604 pr_err("ERR_MDMA: Failed to get desc\n");
605 midc_desc_put(midc, desc);
606 return NULL;
607}
608
609/**
610 * intel_mid_dma_free_chan_resources - Frees dma resources
611 * @chan: chan requiring attention
612 *
613 * Frees the allocated resources on this DMA chan
614 */
615static void intel_mid_dma_free_chan_resources(struct dma_chan *chan)
616{
617 struct intel_mid_dma_chan *midc = to_intel_mid_dma_chan(chan);
618 struct middma_device *mid = to_middma_device(chan->device);
619 struct intel_mid_dma_desc *desc, *_desc;
620
621 if (true == midc->in_use) {
622 /*trying to free ch in use!!!!!*/
623 pr_err("ERR_MDMA: trying to free ch in use\n");
624 }
625
626 spin_lock_bh(&midc->lock);
627 midc->descs_allocated = 0;
628 list_for_each_entry_safe(desc, _desc, &midc->active_list, desc_node) {
629 list_del(&desc->desc_node);
630 pci_pool_free(mid->dma_pool, desc, desc->txd.phys);
631 }
632 list_for_each_entry_safe(desc, _desc, &midc->free_list, desc_node) {
633 list_del(&desc->desc_node);
634 pci_pool_free(mid->dma_pool, desc, desc->txd.phys);
635 }
636 list_for_each_entry_safe(desc, _desc, &midc->queue, desc_node) {
637 list_del(&desc->desc_node);
638 pci_pool_free(mid->dma_pool, desc, desc->txd.phys);
639 }
640 spin_unlock_bh(&midc->lock);
641 midc->in_use = false;
642 /* Disable CH interrupts */
643 iowrite32(MASK_INTR_REG(midc->ch_id), mid->dma_base + MASK_BLOCK);
644 iowrite32(MASK_INTR_REG(midc->ch_id), mid->dma_base + MASK_ERR);
645}
646
647/**
648 * intel_mid_dma_alloc_chan_resources - Allocate dma resources
649 * @chan: chan requiring attention
650 *
651 * Allocates DMA resources on this chan
652 * Return the descriptors allocated
653 */
654static int intel_mid_dma_alloc_chan_resources(struct dma_chan *chan)
655{
656 struct intel_mid_dma_chan *midc = to_intel_mid_dma_chan(chan);
657 struct middma_device *mid = to_middma_device(chan->device);
658 struct intel_mid_dma_desc *desc;
659 dma_addr_t phys;
660 int i = 0;
661
662
663 /* ASSERT: channel is idle */
664 if (test_ch_en(mid->dma_base, midc->ch_id)) {
665 /*ch is not idle*/
666 pr_err("ERR_MDMA: ch not idle\n");
667 return -EIO;
668 }
669 midc->completed = chan->cookie = 1;
670
671 spin_lock_bh(&midc->lock);
672 while (midc->descs_allocated < DESCS_PER_CHANNEL) {
673 spin_unlock_bh(&midc->lock);
674 desc = pci_pool_alloc(mid->dma_pool, GFP_KERNEL, &phys);
675 if (!desc) {
676 pr_err("ERR_MDMA: desc failed\n");
677 return -ENOMEM;
678 /*check*/
679 }
680 dma_async_tx_descriptor_init(&desc->txd, chan);
681 desc->txd.tx_submit = intel_mid_dma_tx_submit;
682 desc->txd.flags = DMA_CTRL_ACK;
683 desc->txd.phys = phys;
684 spin_lock_bh(&midc->lock);
685 i = ++midc->descs_allocated;
686 list_add_tail(&desc->desc_node, &midc->free_list);
687 }
688 spin_unlock_bh(&midc->lock);
689 midc->in_use = false;
690 pr_debug("MID_DMA: Desc alloc done ret: %d desc\n", i);
691 return i;
692}
693
694/**
695 * midc_handle_error - Handle DMA txn error
696 * @mid: controller where error occured
697 * @midc: chan where error occured
698 *
699 * Scan the descriptor for error
700 */
701static void midc_handle_error(struct middma_device *mid,
702 struct intel_mid_dma_chan *midc)
703{
704 midc_scan_descriptors(mid, midc);
705}
706
707/**
708 * dma_tasklet - DMA interrupt tasklet
709 * @data: tasklet arg (the controller structure)
710 *
711 * Scan the controller for interrupts for completion/error
712 * Clear the interrupt and call for handling completion/error
713 */
714static void dma_tasklet(unsigned long data)
715{
716 struct middma_device *mid = NULL;
717 struct intel_mid_dma_chan *midc = NULL;
718 u32 status;
719 int i;
720
721 mid = (struct middma_device *)data;
722 if (mid == NULL) {
723 pr_err("ERR_MDMA: tasklet Null param\n");
724 return;
725 }
726 pr_debug("MDMA: in tasklet for device %x\n", mid->pci_id);
727 status = ioread32(mid->dma_base + RAW_TFR);
728 pr_debug("MDMA:RAW_TFR %x\n", status);
729 status &= mid->intr_mask;
730 while (status) {
731 /*txn interrupt*/
732 i = get_ch_index(&status, mid->chan_base);
733 if (i < 0) {
734 pr_err("ERR_MDMA:Invalid ch index %x\n", i);
735 return;
736 }
737 midc = &mid->ch[i];
738 if (midc == NULL) {
739 pr_err("ERR_MDMA:Null param midc\n");
740 return;
741 }
742 pr_debug("MDMA:Tx complete interrupt %x, Ch No %d Index %d\n",
743 status, midc->ch_id, i);
744 /*clearing this interrupts first*/
745 iowrite32((1 << midc->ch_id), mid->dma_base + CLEAR_TFR);
746 iowrite32((1 << midc->ch_id), mid->dma_base + CLEAR_BLOCK);
747
748 spin_lock_bh(&midc->lock);
749 midc_scan_descriptors(mid, midc);
750 pr_debug("MDMA:Scan of desc... complete, unmasking\n");
751 iowrite32(UNMASK_INTR_REG(midc->ch_id),
752 mid->dma_base + MASK_TFR);
753 spin_unlock_bh(&midc->lock);
754 }
755
756 status = ioread32(mid->dma_base + RAW_ERR);
757 status &= mid->intr_mask;
758 while (status) {
759 /*err interrupt*/
760 i = get_ch_index(&status, mid->chan_base);
761 if (i < 0) {
762 pr_err("ERR_MDMA:Invalid ch index %x\n", i);
763 return;
764 }
765 midc = &mid->ch[i];
766 if (midc == NULL) {
767 pr_err("ERR_MDMA:Null param midc\n");
768 return;
769 }
770 pr_debug("MDMA:Tx complete interrupt %x, Ch No %d Index %d\n",
771 status, midc->ch_id, i);
772
773 iowrite32((1 << midc->ch_id), mid->dma_base + CLEAR_ERR);
774 spin_lock_bh(&midc->lock);
775 midc_handle_error(mid, midc);
776 iowrite32(UNMASK_INTR_REG(midc->ch_id),
777 mid->dma_base + MASK_ERR);
778 spin_unlock_bh(&midc->lock);
779 }
780 pr_debug("MDMA:Exiting takslet...\n");
781 return;
782}
783
784static void dma_tasklet1(unsigned long data)
785{
786 pr_debug("MDMA:in takslet1...\n");
787 return dma_tasklet(data);
788}
789
790static void dma_tasklet2(unsigned long data)
791{
792 pr_debug("MDMA:in takslet2...\n");
793 return dma_tasklet(data);
794}
795
796/**
797 * intel_mid_dma_interrupt - DMA ISR
798 * @irq: IRQ where interrupt occurred
799 * @data: ISR cllback data (the controller structure)
800 *
801 * See if this is our interrupt if so then schedule the tasklet
802 * otherwise ignore
803 */
804static irqreturn_t intel_mid_dma_interrupt(int irq, void *data)
805{
806 struct middma_device *mid = data;
807 u32 status;
808 int call_tasklet = 0;
809
810 /*DMA Interrupt*/
811 pr_debug("MDMA:Got an interrupt on irq %d\n", irq);
812 if (!mid) {
813 pr_err("ERR_MDMA:null pointer mid\n");
814 return -EINVAL;
815 }
816
817 status = ioread32(mid->dma_base + RAW_TFR);
818 pr_debug("MDMA: Status %x, Mask %x\n", status, mid->intr_mask);
819 status &= mid->intr_mask;
820 if (status) {
821 /*need to disable intr*/
822 iowrite32((status << 8), mid->dma_base + MASK_TFR);
823 pr_debug("MDMA: Calling tasklet %x\n", status);
824 call_tasklet = 1;
825 }
826 status = ioread32(mid->dma_base + RAW_ERR);
827 status &= mid->intr_mask;
828 if (status) {
829 iowrite32(MASK_INTR_REG(status), mid->dma_base + MASK_ERR);
830 call_tasklet = 1;
831 }
832 if (call_tasklet)
833 tasklet_schedule(&mid->tasklet);
834
835 return IRQ_HANDLED;
836}
837
838static irqreturn_t intel_mid_dma_interrupt1(int irq, void *data)
839{
840 return intel_mid_dma_interrupt(irq, data);
841}
842
843static irqreturn_t intel_mid_dma_interrupt2(int irq, void *data)
844{
845 return intel_mid_dma_interrupt(irq, data);
846}
847
848/**
849 * mid_setup_dma - Setup the DMA controller
850 * @pdev: Controller PCI device structure
851 *
852 * Initilize the DMA controller, channels, registers with DMA engine,
853 * ISR. Initilize DMA controller channels.
854 */
855static int mid_setup_dma(struct pci_dev *pdev)
856{
857 struct middma_device *dma = pci_get_drvdata(pdev);
858 int err, i;
859 unsigned int irq_level;
860
861 /* DMA coherent memory pool for DMA descriptor allocations */
862 dma->dma_pool = pci_pool_create("intel_mid_dma_desc_pool", pdev,
863 sizeof(struct intel_mid_dma_desc),
864 32, 0);
865 if (NULL == dma->dma_pool) {
866 pr_err("ERR_MDMA:pci_pool_create failed\n");
867 err = -ENOMEM;
868 kfree(dma);
869 goto err_dma_pool;
870 }
871
872 INIT_LIST_HEAD(&dma->common.channels);
873 dma->pci_id = pdev->device;
874 if (dma->pimr_mask) {
875 dma->mask_reg = ioremap(LNW_PERIPHRAL_MASK_BASE,
876 LNW_PERIPHRAL_MASK_SIZE);
877 if (dma->mask_reg == NULL) {
878 pr_err("ERR_MDMA:Cant map periphral intr space !!\n");
879 return -ENOMEM;
880 }
881 } else
882 dma->mask_reg = NULL;
883
884 pr_debug("MDMA:Adding %d channel for this controller\n", dma->max_chan);
885 /*init CH structures*/
886 dma->intr_mask = 0;
887 for (i = 0; i < dma->max_chan; i++) {
888 struct intel_mid_dma_chan *midch = &dma->ch[i];
889
890 midch->chan.device = &dma->common;
891 midch->chan.cookie = 1;
892 midch->chan.chan_id = i;
893 midch->ch_id = dma->chan_base + i;
894 pr_debug("MDMA:Init CH %d, ID %d\n", i, midch->ch_id);
895
896 midch->dma_base = dma->dma_base;
897 midch->ch_regs = dma->dma_base + DMA_CH_SIZE * midch->ch_id;
898 midch->dma = dma;
899 dma->intr_mask |= 1 << (dma->chan_base + i);
900 spin_lock_init(&midch->lock);
901
902 INIT_LIST_HEAD(&midch->active_list);
903 INIT_LIST_HEAD(&midch->queue);
904 INIT_LIST_HEAD(&midch->free_list);
905 /*mask interrupts*/
906 iowrite32(MASK_INTR_REG(midch->ch_id),
907 dma->dma_base + MASK_BLOCK);
908 iowrite32(MASK_INTR_REG(midch->ch_id),
909 dma->dma_base + MASK_SRC_TRAN);
910 iowrite32(MASK_INTR_REG(midch->ch_id),
911 dma->dma_base + MASK_DST_TRAN);
912 iowrite32(MASK_INTR_REG(midch->ch_id),
913 dma->dma_base + MASK_ERR);
914 iowrite32(MASK_INTR_REG(midch->ch_id),
915 dma->dma_base + MASK_TFR);
916
917 disable_dma_interrupt(midch);
918 list_add_tail(&midch->chan.device_node, &dma->common.channels);
919 }
920 pr_debug("MDMA: Calc Mask as %x for this controller\n", dma->intr_mask);
921
922 /*init dma structure*/
923 dma_cap_zero(dma->common.cap_mask);
924 dma_cap_set(DMA_MEMCPY, dma->common.cap_mask);
925 dma_cap_set(DMA_SLAVE, dma->common.cap_mask);
926 dma_cap_set(DMA_PRIVATE, dma->common.cap_mask);
927 dma->common.dev = &pdev->dev;
928 dma->common.chancnt = dma->max_chan;
929
930 dma->common.device_alloc_chan_resources =
931 intel_mid_dma_alloc_chan_resources;
932 dma->common.device_free_chan_resources =
933 intel_mid_dma_free_chan_resources;
934
935 dma->common.device_tx_status = intel_mid_dma_tx_status;
936 dma->common.device_prep_dma_memcpy = intel_mid_dma_prep_memcpy;
937 dma->common.device_issue_pending = intel_mid_dma_issue_pending;
938 dma->common.device_prep_slave_sg = intel_mid_dma_prep_slave_sg;
939 dma->common.device_control = intel_mid_dma_device_control;
940
941 /*enable dma cntrl*/
942 iowrite32(REG_BIT0, dma->dma_base + DMA_CFG);
943
944 /*register irq */
945 if (dma->pimr_mask) {
946 irq_level = IRQF_SHARED;
947 pr_debug("MDMA:Requesting irq shared for DMAC1\n");
948 err = request_irq(pdev->irq, intel_mid_dma_interrupt1,
949 IRQF_SHARED, "INTEL_MID_DMAC1", dma);
950 if (0 != err)
951 goto err_irq;
952 } else {
953 dma->intr_mask = 0x03;
954 irq_level = 0;
955 pr_debug("MDMA:Requesting irq for DMAC2\n");
956 err = request_irq(pdev->irq, intel_mid_dma_interrupt2,
957 0, "INTEL_MID_DMAC2", dma);
958 if (0 != err)
959 goto err_irq;
960 }
961 /*register device w/ engine*/
962 err = dma_async_device_register(&dma->common);
963 if (0 != err) {
964 pr_err("ERR_MDMA:device_register failed: %d\n", err);
965 goto err_engine;
966 }
967 if (dma->pimr_mask) {
968 pr_debug("setting up tasklet1 for DMAC1\n");
969 tasklet_init(&dma->tasklet, dma_tasklet1, (unsigned long)dma);
970 } else {
971 pr_debug("setting up tasklet2 for DMAC2\n");
972 tasklet_init(&dma->tasklet, dma_tasklet2, (unsigned long)dma);
973 }
974 return 0;
975
976err_engine:
977 free_irq(pdev->irq, dma);
978err_irq:
979 pci_pool_destroy(dma->dma_pool);
980 kfree(dma);
981err_dma_pool:
982 pr_err("ERR_MDMA:setup_dma failed: %d\n", err);
983 return err;
984
985}
986
987/**
988 * middma_shutdown - Shutdown the DMA controller
989 * @pdev: Controller PCI device structure
990 *
991 * Called by remove
992 * Unregister DMa controller, clear all structures and free interrupt
993 */
994static void middma_shutdown(struct pci_dev *pdev)
995{
996 struct middma_device *device = pci_get_drvdata(pdev);
997
998 dma_async_device_unregister(&device->common);
999 pci_pool_destroy(device->dma_pool);
1000 if (device->mask_reg)
1001 iounmap(device->mask_reg);
1002 if (device->dma_base)
1003 iounmap(device->dma_base);
1004 free_irq(pdev->irq, device);
1005 return;
1006}
1007
1008/**
1009 * intel_mid_dma_probe - PCI Probe
1010 * @pdev: Controller PCI device structure
1011 * @id: pci device id structure
1012 *
1013 * Initilize the PCI device, map BARs, query driver data.
1014 * Call setup_dma to complete contoller and chan initilzation
1015 */
1016static int __devinit intel_mid_dma_probe(struct pci_dev *pdev,
1017 const struct pci_device_id *id)
1018{
1019 struct middma_device *device;
1020 u32 base_addr, bar_size;
1021 struct intel_mid_dma_probe_info *info;
1022 int err;
1023
1024 pr_debug("MDMA: probe for %x\n", pdev->device);
1025 info = (void *)id->driver_data;
1026 pr_debug("MDMA: CH %d, base %d, block len %d, Periphral mask %x\n",
1027 info->max_chan, info->ch_base,
1028 info->block_size, info->pimr_mask);
1029
1030 err = pci_enable_device(pdev);
1031 if (err)
1032 goto err_enable_device;
1033
1034 err = pci_request_regions(pdev, "intel_mid_dmac");
1035 if (err)
1036 goto err_request_regions;
1037
1038 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
1039 if (err)
1040 goto err_set_dma_mask;
1041
1042 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
1043 if (err)
1044 goto err_set_dma_mask;
1045
1046 device = kzalloc(sizeof(*device), GFP_KERNEL);
1047 if (!device) {
1048 pr_err("ERR_MDMA:kzalloc failed probe\n");
1049 err = -ENOMEM;
1050 goto err_kzalloc;
1051 }
1052 device->pdev = pci_dev_get(pdev);
1053
1054 base_addr = pci_resource_start(pdev, 0);
1055 bar_size = pci_resource_len(pdev, 0);
1056 device->dma_base = ioremap_nocache(base_addr, DMA_REG_SIZE);
1057 if (!device->dma_base) {
1058 pr_err("ERR_MDMA:ioremap failed\n");
1059 err = -ENOMEM;
1060 goto err_ioremap;
1061 }
1062 pci_set_drvdata(pdev, device);
1063 pci_set_master(pdev);
1064 device->max_chan = info->max_chan;
1065 device->chan_base = info->ch_base;
1066 device->block_size = info->block_size;
1067 device->pimr_mask = info->pimr_mask;
1068
1069 err = mid_setup_dma(pdev);
1070 if (err)
1071 goto err_dma;
1072
1073 return 0;
1074
1075err_dma:
1076 iounmap(device->dma_base);
1077err_ioremap:
1078 pci_dev_put(pdev);
1079 kfree(device);
1080err_kzalloc:
1081err_set_dma_mask:
1082 pci_release_regions(pdev);
1083 pci_disable_device(pdev);
1084err_request_regions:
1085err_enable_device:
1086 pr_err("ERR_MDMA:Probe failed %d\n", err);
1087 return err;
1088}
1089
1090/**
1091 * intel_mid_dma_remove - PCI remove
1092 * @pdev: Controller PCI device structure
1093 *
1094 * Free up all resources and data
1095 * Call shutdown_dma to complete contoller and chan cleanup
1096 */
1097static void __devexit intel_mid_dma_remove(struct pci_dev *pdev)
1098{
1099 struct middma_device *device = pci_get_drvdata(pdev);
1100 middma_shutdown(pdev);
1101 pci_dev_put(pdev);
1102 kfree(device);
1103 pci_release_regions(pdev);
1104 pci_disable_device(pdev);
1105}
1106
1107/******************************************************************************
1108* PCI stuff
1109*/
1110static struct pci_device_id intel_mid_dma_ids[] = {
1111 { PCI_VDEVICE(INTEL, INTEL_MID_DMAC1_ID), INFO(2, 6, 4095, 0x200020)},
1112 { PCI_VDEVICE(INTEL, INTEL_MID_DMAC2_ID), INFO(2, 0, 2047, 0)},
1113 { PCI_VDEVICE(INTEL, INTEL_MID_GP_DMAC2_ID), INFO(2, 0, 2047, 0)},
1114 { PCI_VDEVICE(INTEL, INTEL_MFLD_DMAC1_ID), INFO(4, 0, 4095, 0x400040)},
1115 { 0, }
1116};
1117MODULE_DEVICE_TABLE(pci, intel_mid_dma_ids);
1118
1119static struct pci_driver intel_mid_dma_pci = {
1120 .name = "Intel MID DMA",
1121 .id_table = intel_mid_dma_ids,
1122 .probe = intel_mid_dma_probe,
1123 .remove = __devexit_p(intel_mid_dma_remove),
1124};
1125
1126static int __init intel_mid_dma_init(void)
1127{
1128 pr_debug("INFO_MDMA: LNW DMA Driver Version %s\n",
1129 INTEL_MID_DMA_DRIVER_VERSION);
1130 return pci_register_driver(&intel_mid_dma_pci);
1131}
1132fs_initcall(intel_mid_dma_init);
1133
1134static void __exit intel_mid_dma_exit(void)
1135{
1136 pci_unregister_driver(&intel_mid_dma_pci);
1137}
1138module_exit(intel_mid_dma_exit);
1139
1140MODULE_AUTHOR("Vinod Koul <vinod.koul@intel.com>");
1141MODULE_DESCRIPTION("Intel (R) MID DMAC Driver");
1142MODULE_LICENSE("GPL v2");
1143MODULE_VERSION(INTEL_MID_DMA_DRIVER_VERSION);
diff --git a/drivers/dma/intel_mid_dma_regs.h b/drivers/dma/intel_mid_dma_regs.h
new file mode 100644
index 000000000000..d81aa658ab09
--- /dev/null
+++ b/drivers/dma/intel_mid_dma_regs.h
@@ -0,0 +1,260 @@
1/*
2 * intel_mid_dma_regs.h - Intel MID DMA Drivers
3 *
4 * Copyright (C) 2008-10 Intel Corp
5 * Author: Vinod Koul <vinod.koul@intel.com>
6 * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation; version 2 of the License.
11 *
12 * This program is distributed in the hope that it will be useful, but
13 * WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License along
18 * with this program; if not, write to the Free Software Foundation, Inc.,
19 * 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA.
20 *
21 * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
22 *
23 *
24 */
25#ifndef __INTEL_MID_DMAC_REGS_H__
26#define __INTEL_MID_DMAC_REGS_H__
27
28#include <linux/dmaengine.h>
29#include <linux/dmapool.h>
30#include <linux/pci_ids.h>
31
32#define INTEL_MID_DMA_DRIVER_VERSION "1.0.5"
33
34#define REG_BIT0 0x00000001
35#define REG_BIT8 0x00000100
36
37#define UNMASK_INTR_REG(chan_num) \
38 ((REG_BIT0 << chan_num) | (REG_BIT8 << chan_num))
39#define MASK_INTR_REG(chan_num) (REG_BIT8 << chan_num)
40
41#define ENABLE_CHANNEL(chan_num) \
42 ((REG_BIT0 << chan_num) | (REG_BIT8 << chan_num))
43
44#define DESCS_PER_CHANNEL 16
45/*DMA Registers*/
46/*registers associated with channel programming*/
47#define DMA_REG_SIZE 0x400
48#define DMA_CH_SIZE 0x58
49
50/*CH X REG = (DMA_CH_SIZE)*CH_NO + REG*/
51#define SAR 0x00 /* Source Address Register*/
52#define DAR 0x08 /* Destination Address Register*/
53#define CTL_LOW 0x18 /* Control Register*/
54#define CTL_HIGH 0x1C /* Control Register*/
55#define CFG_LOW 0x40 /* Configuration Register Low*/
56#define CFG_HIGH 0x44 /* Configuration Register high*/
57
58#define STATUS_TFR 0x2E8
59#define STATUS_BLOCK 0x2F0
60#define STATUS_ERR 0x308
61
62#define RAW_TFR 0x2C0
63#define RAW_BLOCK 0x2C8
64#define RAW_ERR 0x2E0
65
66#define MASK_TFR 0x310
67#define MASK_BLOCK 0x318
68#define MASK_SRC_TRAN 0x320
69#define MASK_DST_TRAN 0x328
70#define MASK_ERR 0x330
71
72#define CLEAR_TFR 0x338
73#define CLEAR_BLOCK 0x340
74#define CLEAR_SRC_TRAN 0x348
75#define CLEAR_DST_TRAN 0x350
76#define CLEAR_ERR 0x358
77
78#define INTR_STATUS 0x360
79#define DMA_CFG 0x398
80#define DMA_CHAN_EN 0x3A0
81
82/*DMA channel control registers*/
83union intel_mid_dma_ctl_lo {
84 struct {
85 u32 int_en:1; /*enable or disable interrupts*/
86 /*should be 0*/
87 u32 dst_tr_width:3; /*destination transfer width*/
88 /*usually 32 bits = 010*/
89 u32 src_tr_width:3; /*source transfer width*/
90 /*usually 32 bits = 010*/
91 u32 dinc:2; /*destination address inc/dec*/
92 /*For mem:INC=00, Periphral NoINC=11*/
93 u32 sinc:2; /*source address inc or dec, as above*/
94 u32 dst_msize:3; /*destination burst transaction length*/
95 /*always = 16 ie 011*/
96 u32 src_msize:3; /*source burst transaction length*/
97 /*always = 16 ie 011*/
98 u32 reser1:3;
99 u32 tt_fc:3; /*transfer type and flow controller*/
100 /*M-M = 000
101 P-M = 010
102 M-P = 001*/
103 u32 dms:2; /*destination master select = 0*/
104 u32 sms:2; /*source master select = 0*/
105 u32 llp_dst_en:1; /*enable/disable destination LLP = 0*/
106 u32 llp_src_en:1; /*enable/disable source LLP = 0*/
107 u32 reser2:3;
108 } ctlx;
109 u32 ctl_lo;
110};
111
112union intel_mid_dma_ctl_hi {
113 struct {
114 u32 block_ts:12; /*block transfer size*/
115 /*configured by DMAC*/
116 u32 reser:20;
117 } ctlx;
118 u32 ctl_hi;
119
120};
121
122/*DMA channel configuration registers*/
123union intel_mid_dma_cfg_lo {
124 struct {
125 u32 reser1:5;
126 u32 ch_prior:3; /*channel priority = 0*/
127 u32 ch_susp:1; /*channel suspend = 0*/
128 u32 fifo_empty:1; /*FIFO empty or not R bit = 0*/
129 u32 hs_sel_dst:1; /*select HW/SW destn handshaking*/
130 /*HW = 0, SW = 1*/
131 u32 hs_sel_src:1; /*select HW/SW src handshaking*/
132 u32 reser2:6;
133 u32 dst_hs_pol:1; /*dest HS interface polarity*/
134 u32 src_hs_pol:1; /*src HS interface polarity*/
135 u32 max_abrst:10; /*max AMBA burst len = 0 (no sw limit*/
136 u32 reload_src:1; /*auto reload src addr =1 if src is P*/
137 u32 reload_dst:1; /*AR destn addr =1 if dstn is P*/
138 } cfgx;
139 u32 cfg_lo;
140};
141
142union intel_mid_dma_cfg_hi {
143 struct {
144 u32 fcmode:1; /*flow control mode = 1*/
145 u32 fifo_mode:1; /*FIFO mode select = 1*/
146 u32 protctl:3; /*protection control = 0*/
147 u32 rsvd:2;
148 u32 src_per:4; /*src hw HS interface*/
149 u32 dst_per:4; /*dstn hw HS interface*/
150 u32 reser2:17;
151 } cfgx;
152 u32 cfg_hi;
153};
154
155/**
156 * struct intel_mid_dma_chan - internal mid representation of a DMA channel
157 * @chan: dma_chan strcture represetation for mid chan
158 * @ch_regs: MMIO register space pointer to channel register
159 * @dma_base: MMIO register space DMA engine base pointer
160 * @ch_id: DMA channel id
161 * @lock: channel spinlock
162 * @completed: DMA cookie
163 * @active_list: current active descriptors
164 * @queue: current queued up descriptors
165 * @free_list: current free descriptors
166 * @slave: dma slave struture
167 * @descs_allocated: total number of decsiptors allocated
168 * @dma: dma device struture pointer
169 * @in_use: bool representing if ch is in use or not
170 */
171struct intel_mid_dma_chan {
172 struct dma_chan chan;
173 void __iomem *ch_regs;
174 void __iomem *dma_base;
175 int ch_id;
176 spinlock_t lock;
177 dma_cookie_t completed;
178 struct list_head active_list;
179 struct list_head queue;
180 struct list_head free_list;
181 struct intel_mid_dma_slave *slave;
182 unsigned int descs_allocated;
183 struct middma_device *dma;
184 bool in_use;
185};
186
187static inline struct intel_mid_dma_chan *to_intel_mid_dma_chan(
188 struct dma_chan *chan)
189{
190 return container_of(chan, struct intel_mid_dma_chan, chan);
191}
192
193/**
194 * struct middma_device - internal representation of a DMA device
195 * @pdev: PCI device
196 * @dma_base: MMIO register space pointer of DMA
197 * @dma_pool: for allocating DMA descriptors
198 * @common: embedded struct dma_device
199 * @tasklet: dma tasklet for processing interrupts
200 * @ch: per channel data
201 * @pci_id: DMA device PCI ID
202 * @intr_mask: Interrupt mask to be used
203 * @mask_reg: MMIO register for periphral mask
204 * @chan_base: Base ch index (read from driver data)
205 * @max_chan: max number of chs supported (from drv_data)
206 * @block_size: Block size of DMA transfer supported (from drv_data)
207 * @pimr_mask: MMIO register addr for periphral interrupt (from drv_data)
208 */
209struct middma_device {
210 struct pci_dev *pdev;
211 void __iomem *dma_base;
212 struct pci_pool *dma_pool;
213 struct dma_device common;
214 struct tasklet_struct tasklet;
215 struct intel_mid_dma_chan ch[MAX_CHAN];
216 unsigned int pci_id;
217 unsigned int intr_mask;
218 void __iomem *mask_reg;
219 int chan_base;
220 int max_chan;
221 int block_size;
222 unsigned int pimr_mask;
223};
224
225static inline struct middma_device *to_middma_device(struct dma_device *common)
226{
227 return container_of(common, struct middma_device, common);
228}
229
230struct intel_mid_dma_desc {
231 void __iomem *block; /*ch ptr*/
232 struct list_head desc_node;
233 struct dma_async_tx_descriptor txd;
234 size_t len;
235 dma_addr_t sar;
236 dma_addr_t dar;
237 u32 cfg_hi;
238 u32 cfg_lo;
239 u32 ctl_lo;
240 u32 ctl_hi;
241 dma_addr_t next;
242 enum dma_data_direction dirn;
243 enum dma_status status;
244 enum intel_mid_dma_width width; /*width of DMA txn*/
245 enum intel_mid_dma_mode cfg_mode; /*mode configuration*/
246
247};
248
249static inline int test_ch_en(void __iomem *dma, u32 ch_no)
250{
251 u32 en_reg = ioread32(dma + DMA_CHAN_EN);
252 return (en_reg >> ch_no) & 0x1;
253}
254
255static inline struct intel_mid_dma_desc *to_intel_mid_dma_desc
256 (struct dma_async_tx_descriptor *txd)
257{
258 return container_of(txd, struct intel_mid_dma_desc, txd);
259}
260#endif /*__INTEL_MID_DMAC_REGS_H__*/
diff --git a/drivers/dma/ioat/dma.h b/drivers/dma/ioat/dma.h
index 6d3a73b57e54..5216c8a92a21 100644
--- a/drivers/dma/ioat/dma.h
+++ b/drivers/dma/ioat/dma.h
@@ -97,6 +97,7 @@ struct ioat_chan_common {
97 #define IOAT_RESET_PENDING 2 97 #define IOAT_RESET_PENDING 2
98 #define IOAT_KOBJ_INIT_FAIL 3 98 #define IOAT_KOBJ_INIT_FAIL 3
99 #define IOAT_RESHAPE_PENDING 4 99 #define IOAT_RESHAPE_PENDING 4
100 #define IOAT_RUN 5
100 struct timer_list timer; 101 struct timer_list timer;
101 #define COMPLETION_TIMEOUT msecs_to_jiffies(100) 102 #define COMPLETION_TIMEOUT msecs_to_jiffies(100)
102 #define IDLE_TIMEOUT msecs_to_jiffies(2000) 103 #define IDLE_TIMEOUT msecs_to_jiffies(2000)
diff --git a/drivers/dma/ioat/dma_v2.c b/drivers/dma/ioat/dma_v2.c
index 3c8b32a83794..216f9d383b5b 100644
--- a/drivers/dma/ioat/dma_v2.c
+++ b/drivers/dma/ioat/dma_v2.c
@@ -287,7 +287,10 @@ void ioat2_timer_event(unsigned long data)
287 chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET); 287 chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET);
288 dev_err(to_dev(chan), "%s: Channel halted (%x)\n", 288 dev_err(to_dev(chan), "%s: Channel halted (%x)\n",
289 __func__, chanerr); 289 __func__, chanerr);
290 BUG_ON(is_ioat_bug(chanerr)); 290 if (test_bit(IOAT_RUN, &chan->state))
291 BUG_ON(is_ioat_bug(chanerr));
292 else /* we never got off the ground */
293 return;
291 } 294 }
292 295
293 /* if we haven't made progress and we have already 296 /* if we haven't made progress and we have already
@@ -492,6 +495,8 @@ static struct ioat_ring_ent **ioat2_alloc_ring(struct dma_chan *c, int order, gf
492 return ring; 495 return ring;
493} 496}
494 497
498void ioat2_free_chan_resources(struct dma_chan *c);
499
495/* ioat2_alloc_chan_resources - allocate/initialize ioat2 descriptor ring 500/* ioat2_alloc_chan_resources - allocate/initialize ioat2 descriptor ring
496 * @chan: channel to be initialized 501 * @chan: channel to be initialized
497 */ 502 */
@@ -500,6 +505,7 @@ int ioat2_alloc_chan_resources(struct dma_chan *c)
500 struct ioat2_dma_chan *ioat = to_ioat2_chan(c); 505 struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
501 struct ioat_chan_common *chan = &ioat->base; 506 struct ioat_chan_common *chan = &ioat->base;
502 struct ioat_ring_ent **ring; 507 struct ioat_ring_ent **ring;
508 u64 status;
503 int order; 509 int order;
504 510
505 /* have we already been set up? */ 511 /* have we already been set up? */
@@ -540,7 +546,20 @@ int ioat2_alloc_chan_resources(struct dma_chan *c)
540 tasklet_enable(&chan->cleanup_task); 546 tasklet_enable(&chan->cleanup_task);
541 ioat2_start_null_desc(ioat); 547 ioat2_start_null_desc(ioat);
542 548
543 return 1 << ioat->alloc_order; 549 /* check that we got off the ground */
550 udelay(5);
551 status = ioat_chansts(chan);
552 if (is_ioat_active(status) || is_ioat_idle(status)) {
553 set_bit(IOAT_RUN, &chan->state);
554 return 1 << ioat->alloc_order;
555 } else {
556 u32 chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET);
557
558 dev_WARN(to_dev(chan),
559 "failed to start channel chanerr: %#x\n", chanerr);
560 ioat2_free_chan_resources(c);
561 return -EFAULT;
562 }
544} 563}
545 564
546bool reshape_ring(struct ioat2_dma_chan *ioat, int order) 565bool reshape_ring(struct ioat2_dma_chan *ioat, int order)
@@ -778,6 +797,7 @@ void ioat2_free_chan_resources(struct dma_chan *c)
778 del_timer_sync(&chan->timer); 797 del_timer_sync(&chan->timer);
779 device->cleanup_fn((unsigned long) c); 798 device->cleanup_fn((unsigned long) c);
780 device->reset_hw(chan); 799 device->reset_hw(chan);
800 clear_bit(IOAT_RUN, &chan->state);
781 801
782 spin_lock_bh(&chan->cleanup_lock); 802 spin_lock_bh(&chan->cleanup_lock);
783 spin_lock_bh(&ioat->prep_lock); 803 spin_lock_bh(&ioat->prep_lock);
diff --git a/drivers/dma/ioat/dma_v3.c b/drivers/dma/ioat/dma_v3.c
index 1cdd22e1051b..d0f499098479 100644
--- a/drivers/dma/ioat/dma_v3.c
+++ b/drivers/dma/ioat/dma_v3.c
@@ -361,7 +361,10 @@ static void ioat3_timer_event(unsigned long data)
361 chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET); 361 chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET);
362 dev_err(to_dev(chan), "%s: Channel halted (%x)\n", 362 dev_err(to_dev(chan), "%s: Channel halted (%x)\n",
363 __func__, chanerr); 363 __func__, chanerr);
364 BUG_ON(is_ioat_bug(chanerr)); 364 if (test_bit(IOAT_RUN, &chan->state))
365 BUG_ON(is_ioat_bug(chanerr));
366 else /* we never got off the ground */
367 return;
365 } 368 }
366 369
367 /* if we haven't made progress and we have already 370 /* if we haven't made progress and we have already
diff --git a/drivers/dma/pch_dma.c b/drivers/dma/pch_dma.c
new file mode 100644
index 000000000000..3533948b88ba
--- /dev/null
+++ b/drivers/dma/pch_dma.c
@@ -0,0 +1,957 @@
1/*
2 * Topcliff PCH DMA controller driver
3 * Copyright (c) 2010 Intel Corporation
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License version 2 as
7 * published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write to the Free Software
16 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
17 */
18
19#include <linux/dmaengine.h>
20#include <linux/dma-mapping.h>
21#include <linux/init.h>
22#include <linux/pci.h>
23#include <linux/interrupt.h>
24#include <linux/module.h>
25#include <linux/pch_dma.h>
26
27#define DRV_NAME "pch-dma"
28
29#define DMA_CTL0_DISABLE 0x0
30#define DMA_CTL0_SG 0x1
31#define DMA_CTL0_ONESHOT 0x2
32#define DMA_CTL0_MODE_MASK_BITS 0x3
33#define DMA_CTL0_DIR_SHIFT_BITS 2
34#define DMA_CTL0_BITS_PER_CH 4
35
36#define DMA_CTL2_START_SHIFT_BITS 8
37#define DMA_CTL2_IRQ_ENABLE_MASK ((1UL << DMA_CTL2_START_SHIFT_BITS) - 1)
38
39#define DMA_STATUS_IDLE 0x0
40#define DMA_STATUS_DESC_READ 0x1
41#define DMA_STATUS_WAIT 0x2
42#define DMA_STATUS_ACCESS 0x3
43#define DMA_STATUS_BITS_PER_CH 2
44#define DMA_STATUS_MASK_BITS 0x3
45#define DMA_STATUS_SHIFT_BITS 16
46#define DMA_STATUS_IRQ(x) (0x1 << (x))
47#define DMA_STATUS_ERR(x) (0x1 << ((x) + 8))
48
49#define DMA_DESC_WIDTH_SHIFT_BITS 12
50#define DMA_DESC_WIDTH_1_BYTE (0x3 << DMA_DESC_WIDTH_SHIFT_BITS)
51#define DMA_DESC_WIDTH_2_BYTES (0x2 << DMA_DESC_WIDTH_SHIFT_BITS)
52#define DMA_DESC_WIDTH_4_BYTES (0x0 << DMA_DESC_WIDTH_SHIFT_BITS)
53#define DMA_DESC_MAX_COUNT_1_BYTE 0x3FF
54#define DMA_DESC_MAX_COUNT_2_BYTES 0x3FF
55#define DMA_DESC_MAX_COUNT_4_BYTES 0x7FF
56#define DMA_DESC_END_WITHOUT_IRQ 0x0
57#define DMA_DESC_END_WITH_IRQ 0x1
58#define DMA_DESC_FOLLOW_WITHOUT_IRQ 0x2
59#define DMA_DESC_FOLLOW_WITH_IRQ 0x3
60
61#define MAX_CHAN_NR 8
62
63static unsigned int init_nr_desc_per_channel = 64;
64module_param(init_nr_desc_per_channel, uint, 0644);
65MODULE_PARM_DESC(init_nr_desc_per_channel,
66 "initial descriptors per channel (default: 64)");
67
68struct pch_dma_desc_regs {
69 u32 dev_addr;
70 u32 mem_addr;
71 u32 size;
72 u32 next;
73};
74
75struct pch_dma_regs {
76 u32 dma_ctl0;
77 u32 dma_ctl1;
78 u32 dma_ctl2;
79 u32 reserved1;
80 u32 dma_sts0;
81 u32 dma_sts1;
82 u32 reserved2;
83 u32 reserved3;
84 struct pch_dma_desc_regs desc[0];
85};
86
87struct pch_dma_desc {
88 struct pch_dma_desc_regs regs;
89 struct dma_async_tx_descriptor txd;
90 struct list_head desc_node;
91 struct list_head tx_list;
92};
93
94struct pch_dma_chan {
95 struct dma_chan chan;
96 void __iomem *membase;
97 enum dma_data_direction dir;
98 struct tasklet_struct tasklet;
99 unsigned long err_status;
100
101 spinlock_t lock;
102
103 dma_cookie_t completed_cookie;
104 struct list_head active_list;
105 struct list_head queue;
106 struct list_head free_list;
107 unsigned int descs_allocated;
108};
109
110#define PDC_DEV_ADDR 0x00
111#define PDC_MEM_ADDR 0x04
112#define PDC_SIZE 0x08
113#define PDC_NEXT 0x0C
114
115#define channel_readl(pdc, name) \
116 readl((pdc)->membase + PDC_##name)
117#define channel_writel(pdc, name, val) \
118 writel((val), (pdc)->membase + PDC_##name)
119
120struct pch_dma {
121 struct dma_device dma;
122 void __iomem *membase;
123 struct pci_pool *pool;
124 struct pch_dma_regs regs;
125 struct pch_dma_desc_regs ch_regs[MAX_CHAN_NR];
126 struct pch_dma_chan channels[0];
127};
128
129#define PCH_DMA_CTL0 0x00
130#define PCH_DMA_CTL1 0x04
131#define PCH_DMA_CTL2 0x08
132#define PCH_DMA_STS0 0x10
133#define PCH_DMA_STS1 0x14
134
135#define dma_readl(pd, name) \
136 readl((pd)->membase + PCH_DMA_##name)
137#define dma_writel(pd, name, val) \
138 writel((val), (pd)->membase + PCH_DMA_##name)
139
140static inline struct pch_dma_desc *to_pd_desc(struct dma_async_tx_descriptor *txd)
141{
142 return container_of(txd, struct pch_dma_desc, txd);
143}
144
145static inline struct pch_dma_chan *to_pd_chan(struct dma_chan *chan)
146{
147 return container_of(chan, struct pch_dma_chan, chan);
148}
149
150static inline struct pch_dma *to_pd(struct dma_device *ddev)
151{
152 return container_of(ddev, struct pch_dma, dma);
153}
154
155static inline struct device *chan2dev(struct dma_chan *chan)
156{
157 return &chan->dev->device;
158}
159
160static inline struct device *chan2parent(struct dma_chan *chan)
161{
162 return chan->dev->device.parent;
163}
164
165static inline struct pch_dma_desc *pdc_first_active(struct pch_dma_chan *pd_chan)
166{
167 return list_first_entry(&pd_chan->active_list,
168 struct pch_dma_desc, desc_node);
169}
170
171static inline struct pch_dma_desc *pdc_first_queued(struct pch_dma_chan *pd_chan)
172{
173 return list_first_entry(&pd_chan->queue,
174 struct pch_dma_desc, desc_node);
175}
176
177static void pdc_enable_irq(struct dma_chan *chan, int enable)
178{
179 struct pch_dma *pd = to_pd(chan->device);
180 u32 val;
181
182 val = dma_readl(pd, CTL2);
183
184 if (enable)
185 val |= 0x1 << chan->chan_id;
186 else
187 val &= ~(0x1 << chan->chan_id);
188
189 dma_writel(pd, CTL2, val);
190
191 dev_dbg(chan2dev(chan), "pdc_enable_irq: chan %d -> %x\n",
192 chan->chan_id, val);
193}
194
195static void pdc_set_dir(struct dma_chan *chan)
196{
197 struct pch_dma_chan *pd_chan = to_pd_chan(chan);
198 struct pch_dma *pd = to_pd(chan->device);
199 u32 val;
200
201 val = dma_readl(pd, CTL0);
202
203 if (pd_chan->dir == DMA_TO_DEVICE)
204 val |= 0x1 << (DMA_CTL0_BITS_PER_CH * chan->chan_id +
205 DMA_CTL0_DIR_SHIFT_BITS);
206 else
207 val &= ~(0x1 << (DMA_CTL0_BITS_PER_CH * chan->chan_id +
208 DMA_CTL0_DIR_SHIFT_BITS));
209
210 dma_writel(pd, CTL0, val);
211
212 dev_dbg(chan2dev(chan), "pdc_set_dir: chan %d -> %x\n",
213 chan->chan_id, val);
214}
215
216static void pdc_set_mode(struct dma_chan *chan, u32 mode)
217{
218 struct pch_dma *pd = to_pd(chan->device);
219 u32 val;
220
221 val = dma_readl(pd, CTL0);
222
223 val &= ~(DMA_CTL0_MODE_MASK_BITS <<
224 (DMA_CTL0_BITS_PER_CH * chan->chan_id));
225 val |= mode << (DMA_CTL0_BITS_PER_CH * chan->chan_id);
226
227 dma_writel(pd, CTL0, val);
228
229 dev_dbg(chan2dev(chan), "pdc_set_mode: chan %d -> %x\n",
230 chan->chan_id, val);
231}
232
233static u32 pdc_get_status(struct pch_dma_chan *pd_chan)
234{
235 struct pch_dma *pd = to_pd(pd_chan->chan.device);
236 u32 val;
237
238 val = dma_readl(pd, STS0);
239 return DMA_STATUS_MASK_BITS & (val >> (DMA_STATUS_SHIFT_BITS +
240 DMA_STATUS_BITS_PER_CH * pd_chan->chan.chan_id));
241}
242
243static bool pdc_is_idle(struct pch_dma_chan *pd_chan)
244{
245 if (pdc_get_status(pd_chan) == DMA_STATUS_IDLE)
246 return true;
247 else
248 return false;
249}
250
251static void pdc_dostart(struct pch_dma_chan *pd_chan, struct pch_dma_desc* desc)
252{
253 struct pch_dma *pd = to_pd(pd_chan->chan.device);
254 u32 val;
255
256 if (!pdc_is_idle(pd_chan)) {
257 dev_err(chan2dev(&pd_chan->chan),
258 "BUG: Attempt to start non-idle channel\n");
259 return;
260 }
261
262 channel_writel(pd_chan, DEV_ADDR, desc->regs.dev_addr);
263 channel_writel(pd_chan, MEM_ADDR, desc->regs.mem_addr);
264 channel_writel(pd_chan, SIZE, desc->regs.size);
265 channel_writel(pd_chan, NEXT, desc->regs.next);
266
267 dev_dbg(chan2dev(&pd_chan->chan), "chan %d -> dev_addr: %x\n",
268 pd_chan->chan.chan_id, desc->regs.dev_addr);
269 dev_dbg(chan2dev(&pd_chan->chan), "chan %d -> mem_addr: %x\n",
270 pd_chan->chan.chan_id, desc->regs.mem_addr);
271 dev_dbg(chan2dev(&pd_chan->chan), "chan %d -> size: %x\n",
272 pd_chan->chan.chan_id, desc->regs.size);
273 dev_dbg(chan2dev(&pd_chan->chan), "chan %d -> next: %x\n",
274 pd_chan->chan.chan_id, desc->regs.next);
275
276 if (list_empty(&desc->tx_list))
277 pdc_set_mode(&pd_chan->chan, DMA_CTL0_ONESHOT);
278 else
279 pdc_set_mode(&pd_chan->chan, DMA_CTL0_SG);
280
281 val = dma_readl(pd, CTL2);
282 val |= 1 << (DMA_CTL2_START_SHIFT_BITS + pd_chan->chan.chan_id);
283 dma_writel(pd, CTL2, val);
284}
285
286static void pdc_chain_complete(struct pch_dma_chan *pd_chan,
287 struct pch_dma_desc *desc)
288{
289 struct dma_async_tx_descriptor *txd = &desc->txd;
290 dma_async_tx_callback callback = txd->callback;
291 void *param = txd->callback_param;
292
293 list_splice_init(&desc->tx_list, &pd_chan->free_list);
294 list_move(&desc->desc_node, &pd_chan->free_list);
295
296 if (callback)
297 callback(param);
298}
299
300static void pdc_complete_all(struct pch_dma_chan *pd_chan)
301{
302 struct pch_dma_desc *desc, *_d;
303 LIST_HEAD(list);
304
305 BUG_ON(!pdc_is_idle(pd_chan));
306
307 if (!list_empty(&pd_chan->queue))
308 pdc_dostart(pd_chan, pdc_first_queued(pd_chan));
309
310 list_splice_init(&pd_chan->active_list, &list);
311 list_splice_init(&pd_chan->queue, &pd_chan->active_list);
312
313 list_for_each_entry_safe(desc, _d, &list, desc_node)
314 pdc_chain_complete(pd_chan, desc);
315}
316
317static void pdc_handle_error(struct pch_dma_chan *pd_chan)
318{
319 struct pch_dma_desc *bad_desc;
320
321 bad_desc = pdc_first_active(pd_chan);
322 list_del(&bad_desc->desc_node);
323
324 list_splice_init(&pd_chan->queue, pd_chan->active_list.prev);
325
326 if (!list_empty(&pd_chan->active_list))
327 pdc_dostart(pd_chan, pdc_first_active(pd_chan));
328
329 dev_crit(chan2dev(&pd_chan->chan), "Bad descriptor submitted\n");
330 dev_crit(chan2dev(&pd_chan->chan), "descriptor cookie: %d\n",
331 bad_desc->txd.cookie);
332
333 pdc_chain_complete(pd_chan, bad_desc);
334}
335
336static void pdc_advance_work(struct pch_dma_chan *pd_chan)
337{
338 if (list_empty(&pd_chan->active_list) ||
339 list_is_singular(&pd_chan->active_list)) {
340 pdc_complete_all(pd_chan);
341 } else {
342 pdc_chain_complete(pd_chan, pdc_first_active(pd_chan));
343 pdc_dostart(pd_chan, pdc_first_active(pd_chan));
344 }
345}
346
347static dma_cookie_t pdc_assign_cookie(struct pch_dma_chan *pd_chan,
348 struct pch_dma_desc *desc)
349{
350 dma_cookie_t cookie = pd_chan->chan.cookie;
351
352 if (++cookie < 0)
353 cookie = 1;
354
355 pd_chan->chan.cookie = cookie;
356 desc->txd.cookie = cookie;
357
358 return cookie;
359}
360
361static dma_cookie_t pd_tx_submit(struct dma_async_tx_descriptor *txd)
362{
363 struct pch_dma_desc *desc = to_pd_desc(txd);
364 struct pch_dma_chan *pd_chan = to_pd_chan(txd->chan);
365 dma_cookie_t cookie;
366
367 spin_lock_bh(&pd_chan->lock);
368 cookie = pdc_assign_cookie(pd_chan, desc);
369
370 if (list_empty(&pd_chan->active_list)) {
371 list_add_tail(&desc->desc_node, &pd_chan->active_list);
372 pdc_dostart(pd_chan, desc);
373 } else {
374 list_add_tail(&desc->desc_node, &pd_chan->queue);
375 }
376
377 spin_unlock_bh(&pd_chan->lock);
378 return 0;
379}
380
381static struct pch_dma_desc *pdc_alloc_desc(struct dma_chan *chan, gfp_t flags)
382{
383 struct pch_dma_desc *desc = NULL;
384 struct pch_dma *pd = to_pd(chan->device);
385 dma_addr_t addr;
386
387 desc = pci_pool_alloc(pd->pool, GFP_KERNEL, &addr);
388 if (desc) {
389 memset(desc, 0, sizeof(struct pch_dma_desc));
390 INIT_LIST_HEAD(&desc->tx_list);
391 dma_async_tx_descriptor_init(&desc->txd, chan);
392 desc->txd.tx_submit = pd_tx_submit;
393 desc->txd.flags = DMA_CTRL_ACK;
394 desc->txd.phys = addr;
395 }
396
397 return desc;
398}
399
400static struct pch_dma_desc *pdc_desc_get(struct pch_dma_chan *pd_chan)
401{
402 struct pch_dma_desc *desc, *_d;
403 struct pch_dma_desc *ret = NULL;
404 int i;
405
406 spin_lock_bh(&pd_chan->lock);
407 list_for_each_entry_safe(desc, _d, &pd_chan->free_list, desc_node) {
408 i++;
409 if (async_tx_test_ack(&desc->txd)) {
410 list_del(&desc->desc_node);
411 ret = desc;
412 break;
413 }
414 dev_dbg(chan2dev(&pd_chan->chan), "desc %p not ACKed\n", desc);
415 }
416 spin_unlock_bh(&pd_chan->lock);
417 dev_dbg(chan2dev(&pd_chan->chan), "scanned %d descriptors\n", i);
418
419 if (!ret) {
420 ret = pdc_alloc_desc(&pd_chan->chan, GFP_NOIO);
421 if (ret) {
422 spin_lock_bh(&pd_chan->lock);
423 pd_chan->descs_allocated++;
424 spin_unlock_bh(&pd_chan->lock);
425 } else {
426 dev_err(chan2dev(&pd_chan->chan),
427 "failed to alloc desc\n");
428 }
429 }
430
431 return ret;
432}
433
434static void pdc_desc_put(struct pch_dma_chan *pd_chan,
435 struct pch_dma_desc *desc)
436{
437 if (desc) {
438 spin_lock_bh(&pd_chan->lock);
439 list_splice_init(&desc->tx_list, &pd_chan->free_list);
440 list_add(&desc->desc_node, &pd_chan->free_list);
441 spin_unlock_bh(&pd_chan->lock);
442 }
443}
444
445static int pd_alloc_chan_resources(struct dma_chan *chan)
446{
447 struct pch_dma_chan *pd_chan = to_pd_chan(chan);
448 struct pch_dma_desc *desc;
449 LIST_HEAD(tmp_list);
450 int i;
451
452 if (!pdc_is_idle(pd_chan)) {
453 dev_dbg(chan2dev(chan), "DMA channel not idle ?\n");
454 return -EIO;
455 }
456
457 if (!list_empty(&pd_chan->free_list))
458 return pd_chan->descs_allocated;
459
460 for (i = 0; i < init_nr_desc_per_channel; i++) {
461 desc = pdc_alloc_desc(chan, GFP_KERNEL);
462
463 if (!desc) {
464 dev_warn(chan2dev(chan),
465 "Only allocated %d initial descriptors\n", i);
466 break;
467 }
468
469 list_add_tail(&desc->desc_node, &tmp_list);
470 }
471
472 spin_lock_bh(&pd_chan->lock);
473 list_splice(&tmp_list, &pd_chan->free_list);
474 pd_chan->descs_allocated = i;
475 pd_chan->completed_cookie = chan->cookie = 1;
476 spin_unlock_bh(&pd_chan->lock);
477
478 pdc_enable_irq(chan, 1);
479 pdc_set_dir(chan);
480
481 return pd_chan->descs_allocated;
482}
483
484static void pd_free_chan_resources(struct dma_chan *chan)
485{
486 struct pch_dma_chan *pd_chan = to_pd_chan(chan);
487 struct pch_dma *pd = to_pd(chan->device);
488 struct pch_dma_desc *desc, *_d;
489 LIST_HEAD(tmp_list);
490
491 BUG_ON(!pdc_is_idle(pd_chan));
492 BUG_ON(!list_empty(&pd_chan->active_list));
493 BUG_ON(!list_empty(&pd_chan->queue));
494
495 spin_lock_bh(&pd_chan->lock);
496 list_splice_init(&pd_chan->free_list, &tmp_list);
497 pd_chan->descs_allocated = 0;
498 spin_unlock_bh(&pd_chan->lock);
499
500 list_for_each_entry_safe(desc, _d, &tmp_list, desc_node)
501 pci_pool_free(pd->pool, desc, desc->txd.phys);
502
503 pdc_enable_irq(chan, 0);
504}
505
506static enum dma_status pd_tx_status(struct dma_chan *chan, dma_cookie_t cookie,
507 struct dma_tx_state *txstate)
508{
509 struct pch_dma_chan *pd_chan = to_pd_chan(chan);
510 dma_cookie_t last_used;
511 dma_cookie_t last_completed;
512 int ret;
513
514 spin_lock_bh(&pd_chan->lock);
515 last_completed = pd_chan->completed_cookie;
516 last_used = chan->cookie;
517 spin_unlock_bh(&pd_chan->lock);
518
519 ret = dma_async_is_complete(cookie, last_completed, last_used);
520
521 dma_set_tx_state(txstate, last_completed, last_used, 0);
522
523 return ret;
524}
525
526static void pd_issue_pending(struct dma_chan *chan)
527{
528 struct pch_dma_chan *pd_chan = to_pd_chan(chan);
529
530 if (pdc_is_idle(pd_chan)) {
531 spin_lock_bh(&pd_chan->lock);
532 pdc_advance_work(pd_chan);
533 spin_unlock_bh(&pd_chan->lock);
534 }
535}
536
537static struct dma_async_tx_descriptor *pd_prep_slave_sg(struct dma_chan *chan,
538 struct scatterlist *sgl, unsigned int sg_len,
539 enum dma_data_direction direction, unsigned long flags)
540{
541 struct pch_dma_chan *pd_chan = to_pd_chan(chan);
542 struct pch_dma_slave *pd_slave = chan->private;
543 struct pch_dma_desc *first = NULL;
544 struct pch_dma_desc *prev = NULL;
545 struct pch_dma_desc *desc = NULL;
546 struct scatterlist *sg;
547 dma_addr_t reg;
548 int i;
549
550 if (unlikely(!sg_len)) {
551 dev_info(chan2dev(chan), "prep_slave_sg: length is zero!\n");
552 return NULL;
553 }
554
555 if (direction == DMA_FROM_DEVICE)
556 reg = pd_slave->rx_reg;
557 else if (direction == DMA_TO_DEVICE)
558 reg = pd_slave->tx_reg;
559 else
560 return NULL;
561
562 for_each_sg(sgl, sg, sg_len, i) {
563 desc = pdc_desc_get(pd_chan);
564
565 if (!desc)
566 goto err_desc_get;
567
568 desc->regs.dev_addr = reg;
569 desc->regs.mem_addr = sg_phys(sg);
570 desc->regs.size = sg_dma_len(sg);
571 desc->regs.next = DMA_DESC_FOLLOW_WITHOUT_IRQ;
572
573 switch (pd_slave->width) {
574 case PCH_DMA_WIDTH_1_BYTE:
575 if (desc->regs.size > DMA_DESC_MAX_COUNT_1_BYTE)
576 goto err_desc_get;
577 desc->regs.size |= DMA_DESC_WIDTH_1_BYTE;
578 break;
579 case PCH_DMA_WIDTH_2_BYTES:
580 if (desc->regs.size > DMA_DESC_MAX_COUNT_2_BYTES)
581 goto err_desc_get;
582 desc->regs.size |= DMA_DESC_WIDTH_2_BYTES;
583 break;
584 case PCH_DMA_WIDTH_4_BYTES:
585 if (desc->regs.size > DMA_DESC_MAX_COUNT_4_BYTES)
586 goto err_desc_get;
587 desc->regs.size |= DMA_DESC_WIDTH_4_BYTES;
588 break;
589 default:
590 goto err_desc_get;
591 }
592
593
594 if (!first) {
595 first = desc;
596 } else {
597 prev->regs.next |= desc->txd.phys;
598 list_add_tail(&desc->desc_node, &first->tx_list);
599 }
600
601 prev = desc;
602 }
603
604 if (flags & DMA_PREP_INTERRUPT)
605 desc->regs.next = DMA_DESC_END_WITH_IRQ;
606 else
607 desc->regs.next = DMA_DESC_END_WITHOUT_IRQ;
608
609 first->txd.cookie = -EBUSY;
610 desc->txd.flags = flags;
611
612 return &first->txd;
613
614err_desc_get:
615 dev_err(chan2dev(chan), "failed to get desc or wrong parameters\n");
616 pdc_desc_put(pd_chan, first);
617 return NULL;
618}
619
620static int pd_device_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd,
621 unsigned long arg)
622{
623 struct pch_dma_chan *pd_chan = to_pd_chan(chan);
624 struct pch_dma_desc *desc, *_d;
625 LIST_HEAD(list);
626
627 if (cmd != DMA_TERMINATE_ALL)
628 return -ENXIO;
629
630 spin_lock_bh(&pd_chan->lock);
631
632 pdc_set_mode(&pd_chan->chan, DMA_CTL0_DISABLE);
633
634 list_splice_init(&pd_chan->active_list, &list);
635 list_splice_init(&pd_chan->queue, &list);
636
637 list_for_each_entry_safe(desc, _d, &list, desc_node)
638 pdc_chain_complete(pd_chan, desc);
639
640 spin_unlock_bh(&pd_chan->lock);
641
642
643 return 0;
644}
645
646static void pdc_tasklet(unsigned long data)
647{
648 struct pch_dma_chan *pd_chan = (struct pch_dma_chan *)data;
649
650 if (!pdc_is_idle(pd_chan)) {
651 dev_err(chan2dev(&pd_chan->chan),
652 "BUG: handle non-idle channel in tasklet\n");
653 return;
654 }
655
656 spin_lock_bh(&pd_chan->lock);
657 if (test_and_clear_bit(0, &pd_chan->err_status))
658 pdc_handle_error(pd_chan);
659 else
660 pdc_advance_work(pd_chan);
661 spin_unlock_bh(&pd_chan->lock);
662}
663
664static irqreturn_t pd_irq(int irq, void *devid)
665{
666 struct pch_dma *pd = (struct pch_dma *)devid;
667 struct pch_dma_chan *pd_chan;
668 u32 sts0;
669 int i;
670 int ret = IRQ_NONE;
671
672 sts0 = dma_readl(pd, STS0);
673
674 dev_dbg(pd->dma.dev, "pd_irq sts0: %x\n", sts0);
675
676 for (i = 0; i < pd->dma.chancnt; i++) {
677 pd_chan = &pd->channels[i];
678
679 if (sts0 & DMA_STATUS_IRQ(i)) {
680 if (sts0 & DMA_STATUS_ERR(i))
681 set_bit(0, &pd_chan->err_status);
682
683 tasklet_schedule(&pd_chan->tasklet);
684 ret = IRQ_HANDLED;
685 }
686
687 }
688
689 /* clear interrupt bits in status register */
690 dma_writel(pd, STS0, sts0);
691
692 return ret;
693}
694
695static void pch_dma_save_regs(struct pch_dma *pd)
696{
697 struct pch_dma_chan *pd_chan;
698 struct dma_chan *chan, *_c;
699 int i = 0;
700
701 pd->regs.dma_ctl0 = dma_readl(pd, CTL0);
702 pd->regs.dma_ctl1 = dma_readl(pd, CTL1);
703 pd->regs.dma_ctl2 = dma_readl(pd, CTL2);
704
705 list_for_each_entry_safe(chan, _c, &pd->dma.channels, device_node) {
706 pd_chan = to_pd_chan(chan);
707
708 pd->ch_regs[i].dev_addr = channel_readl(pd_chan, DEV_ADDR);
709 pd->ch_regs[i].mem_addr = channel_readl(pd_chan, MEM_ADDR);
710 pd->ch_regs[i].size = channel_readl(pd_chan, SIZE);
711 pd->ch_regs[i].next = channel_readl(pd_chan, NEXT);
712
713 i++;
714 }
715}
716
717static void pch_dma_restore_regs(struct pch_dma *pd)
718{
719 struct pch_dma_chan *pd_chan;
720 struct dma_chan *chan, *_c;
721 int i = 0;
722
723 dma_writel(pd, CTL0, pd->regs.dma_ctl0);
724 dma_writel(pd, CTL1, pd->regs.dma_ctl1);
725 dma_writel(pd, CTL2, pd->regs.dma_ctl2);
726
727 list_for_each_entry_safe(chan, _c, &pd->dma.channels, device_node) {
728 pd_chan = to_pd_chan(chan);
729
730 channel_writel(pd_chan, DEV_ADDR, pd->ch_regs[i].dev_addr);
731 channel_writel(pd_chan, MEM_ADDR, pd->ch_regs[i].mem_addr);
732 channel_writel(pd_chan, SIZE, pd->ch_regs[i].size);
733 channel_writel(pd_chan, NEXT, pd->ch_regs[i].next);
734
735 i++;
736 }
737}
738
739static int pch_dma_suspend(struct pci_dev *pdev, pm_message_t state)
740{
741 struct pch_dma *pd = pci_get_drvdata(pdev);
742
743 if (pd)
744 pch_dma_save_regs(pd);
745
746 pci_save_state(pdev);
747 pci_disable_device(pdev);
748 pci_set_power_state(pdev, pci_choose_state(pdev, state));
749
750 return 0;
751}
752
753static int pch_dma_resume(struct pci_dev *pdev)
754{
755 struct pch_dma *pd = pci_get_drvdata(pdev);
756 int err;
757
758 pci_set_power_state(pdev, PCI_D0);
759 pci_restore_state(pdev);
760
761 err = pci_enable_device(pdev);
762 if (err) {
763 dev_dbg(&pdev->dev, "failed to enable device\n");
764 return err;
765 }
766
767 if (pd)
768 pch_dma_restore_regs(pd);
769
770 return 0;
771}
772
773static int __devinit pch_dma_probe(struct pci_dev *pdev,
774 const struct pci_device_id *id)
775{
776 struct pch_dma *pd;
777 struct pch_dma_regs *regs;
778 unsigned int nr_channels;
779 int err;
780 int i;
781
782 nr_channels = id->driver_data;
783 pd = kzalloc(sizeof(struct pch_dma)+
784 sizeof(struct pch_dma_chan) * nr_channels, GFP_KERNEL);
785 if (!pd)
786 return -ENOMEM;
787
788 pci_set_drvdata(pdev, pd);
789
790 err = pci_enable_device(pdev);
791 if (err) {
792 dev_err(&pdev->dev, "Cannot enable PCI device\n");
793 goto err_free_mem;
794 }
795
796 if (!(pci_resource_flags(pdev, 1) & IORESOURCE_MEM)) {
797 dev_err(&pdev->dev, "Cannot find proper base address\n");
798 goto err_disable_pdev;
799 }
800
801 err = pci_request_regions(pdev, DRV_NAME);
802 if (err) {
803 dev_err(&pdev->dev, "Cannot obtain PCI resources\n");
804 goto err_disable_pdev;
805 }
806
807 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
808 if (err) {
809 dev_err(&pdev->dev, "Cannot set proper DMA config\n");
810 goto err_free_res;
811 }
812
813 regs = pd->membase = pci_iomap(pdev, 1, 0);
814 if (!pd->membase) {
815 dev_err(&pdev->dev, "Cannot map MMIO registers\n");
816 err = -ENOMEM;
817 goto err_free_res;
818 }
819
820 pci_set_master(pdev);
821
822 err = request_irq(pdev->irq, pd_irq, IRQF_SHARED, DRV_NAME, pd);
823 if (err) {
824 dev_err(&pdev->dev, "Failed to request IRQ\n");
825 goto err_iounmap;
826 }
827
828 pd->pool = pci_pool_create("pch_dma_desc_pool", pdev,
829 sizeof(struct pch_dma_desc), 4, 0);
830 if (!pd->pool) {
831 dev_err(&pdev->dev, "Failed to alloc DMA descriptors\n");
832 err = -ENOMEM;
833 goto err_free_irq;
834 }
835
836 pd->dma.dev = &pdev->dev;
837 pd->dma.chancnt = nr_channels;
838
839 INIT_LIST_HEAD(&pd->dma.channels);
840
841 for (i = 0; i < nr_channels; i++) {
842 struct pch_dma_chan *pd_chan = &pd->channels[i];
843
844 pd_chan->chan.device = &pd->dma;
845 pd_chan->chan.cookie = 1;
846 pd_chan->chan.chan_id = i;
847
848 pd_chan->membase = &regs->desc[i];
849
850 pd_chan->dir = (i % 2) ? DMA_FROM_DEVICE : DMA_TO_DEVICE;
851
852 spin_lock_init(&pd_chan->lock);
853
854 INIT_LIST_HEAD(&pd_chan->active_list);
855 INIT_LIST_HEAD(&pd_chan->queue);
856 INIT_LIST_HEAD(&pd_chan->free_list);
857
858 tasklet_init(&pd_chan->tasklet, pdc_tasklet,
859 (unsigned long)pd_chan);
860 list_add_tail(&pd_chan->chan.device_node, &pd->dma.channels);
861 }
862
863 dma_cap_zero(pd->dma.cap_mask);
864 dma_cap_set(DMA_PRIVATE, pd->dma.cap_mask);
865 dma_cap_set(DMA_SLAVE, pd->dma.cap_mask);
866
867 pd->dma.device_alloc_chan_resources = pd_alloc_chan_resources;
868 pd->dma.device_free_chan_resources = pd_free_chan_resources;
869 pd->dma.device_tx_status = pd_tx_status;
870 pd->dma.device_issue_pending = pd_issue_pending;
871 pd->dma.device_prep_slave_sg = pd_prep_slave_sg;
872 pd->dma.device_control = pd_device_control;
873
874 err = dma_async_device_register(&pd->dma);
875 if (err) {
876 dev_err(&pdev->dev, "Failed to register DMA device\n");
877 goto err_free_pool;
878 }
879
880 return 0;
881
882err_free_pool:
883 pci_pool_destroy(pd->pool);
884err_free_irq:
885 free_irq(pdev->irq, pd);
886err_iounmap:
887 pci_iounmap(pdev, pd->membase);
888err_free_res:
889 pci_release_regions(pdev);
890err_disable_pdev:
891 pci_disable_device(pdev);
892err_free_mem:
893 return err;
894}
895
896static void __devexit pch_dma_remove(struct pci_dev *pdev)
897{
898 struct pch_dma *pd = pci_get_drvdata(pdev);
899 struct pch_dma_chan *pd_chan;
900 struct dma_chan *chan, *_c;
901
902 if (pd) {
903 dma_async_device_unregister(&pd->dma);
904
905 list_for_each_entry_safe(chan, _c, &pd->dma.channels,
906 device_node) {
907 pd_chan = to_pd_chan(chan);
908
909 tasklet_disable(&pd_chan->tasklet);
910 tasklet_kill(&pd_chan->tasklet);
911 }
912
913 pci_pool_destroy(pd->pool);
914 free_irq(pdev->irq, pd);
915 pci_iounmap(pdev, pd->membase);
916 pci_release_regions(pdev);
917 pci_disable_device(pdev);
918 kfree(pd);
919 }
920}
921
922/* PCI Device ID of DMA device */
923#define PCI_DEVICE_ID_PCH_DMA_8CH 0x8810
924#define PCI_DEVICE_ID_PCH_DMA_4CH 0x8815
925
926static const struct pci_device_id pch_dma_id_table[] = {
927 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_PCH_DMA_8CH), 8 },
928 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_PCH_DMA_4CH), 4 },
929};
930
931static struct pci_driver pch_dma_driver = {
932 .name = DRV_NAME,
933 .id_table = pch_dma_id_table,
934 .probe = pch_dma_probe,
935 .remove = __devexit_p(pch_dma_remove),
936#ifdef CONFIG_PM
937 .suspend = pch_dma_suspend,
938 .resume = pch_dma_resume,
939#endif
940};
941
942static int __init pch_dma_init(void)
943{
944 return pci_register_driver(&pch_dma_driver);
945}
946
947static void __exit pch_dma_exit(void)
948{
949 pci_unregister_driver(&pch_dma_driver);
950}
951
952module_init(pch_dma_init);
953module_exit(pch_dma_exit);
954
955MODULE_DESCRIPTION("Topcliff PCH DMA controller driver");
956MODULE_AUTHOR("Yong Wang <yong.y.wang@intel.com>");
957MODULE_LICENSE("GPL v2");
diff --git a/drivers/dma/ste_dma40.c b/drivers/dma/ste_dma40.c
index c426829f6ab8..17e2600a00cf 100644
--- a/drivers/dma/ste_dma40.c
+++ b/drivers/dma/ste_dma40.c
@@ -30,14 +30,16 @@
30/* Maximum iterations taken before giving up suspending a channel */ 30/* Maximum iterations taken before giving up suspending a channel */
31#define D40_SUSPEND_MAX_IT 500 31#define D40_SUSPEND_MAX_IT 500
32 32
33/* Hardware requirement on LCLA alignment */
34#define LCLA_ALIGNMENT 0x40000
35/* Attempts before giving up to trying to get pages that are aligned */
36#define MAX_LCLA_ALLOC_ATTEMPTS 256
37
38/* Bit markings for allocation map */
33#define D40_ALLOC_FREE (1 << 31) 39#define D40_ALLOC_FREE (1 << 31)
34#define D40_ALLOC_PHY (1 << 30) 40#define D40_ALLOC_PHY (1 << 30)
35#define D40_ALLOC_LOG_FREE 0 41#define D40_ALLOC_LOG_FREE 0
36 42
37/* The number of free d40_desc to keep in memory before starting
38 * to kfree() them */
39#define D40_DESC_CACHE_SIZE 50
40
41/* Hardware designer of the block */ 43/* Hardware designer of the block */
42#define D40_PERIPHID2_DESIGNER 0x8 44#define D40_PERIPHID2_DESIGNER 0x8
43 45
@@ -68,9 +70,9 @@ enum d40_command {
68 */ 70 */
69struct d40_lli_pool { 71struct d40_lli_pool {
70 void *base; 72 void *base;
71 int size; 73 int size;
72 /* Space for dst and src, plus an extra for padding */ 74 /* Space for dst and src, plus an extra for padding */
73 u8 pre_alloc_lli[3 * sizeof(struct d40_phy_lli)]; 75 u8 pre_alloc_lli[3 * sizeof(struct d40_phy_lli)];
74}; 76};
75 77
76/** 78/**
@@ -81,9 +83,10 @@ struct d40_lli_pool {
81 * lli_len equals one. 83 * lli_len equals one.
82 * @lli_log: Same as above but for logical channels. 84 * @lli_log: Same as above but for logical channels.
83 * @lli_pool: The pool with two entries pre-allocated. 85 * @lli_pool: The pool with two entries pre-allocated.
84 * @lli_len: Number of LLI's in lli_pool 86 * @lli_len: Number of llis of current descriptor.
85 * @lli_tcount: Number of LLIs processed in the transfer. When equals lli_len 87 * @lli_count: Number of transfered llis.
86 * then this transfer job is done. 88 * @lli_tx_len: Max number of LLIs per transfer, there can be
89 * many transfer for one descriptor.
87 * @txd: DMA engine struct. Used for among other things for communication 90 * @txd: DMA engine struct. Used for among other things for communication
88 * during a transfer. 91 * during a transfer.
89 * @node: List entry. 92 * @node: List entry.
@@ -100,8 +103,9 @@ struct d40_desc {
100 struct d40_log_lli_bidir lli_log; 103 struct d40_log_lli_bidir lli_log;
101 104
102 struct d40_lli_pool lli_pool; 105 struct d40_lli_pool lli_pool;
103 u32 lli_len; 106 int lli_len;
104 u32 lli_tcount; 107 int lli_count;
108 u32 lli_tx_len;
105 109
106 struct dma_async_tx_descriptor txd; 110 struct dma_async_tx_descriptor txd;
107 struct list_head node; 111 struct list_head node;
@@ -113,18 +117,20 @@ struct d40_desc {
113/** 117/**
114 * struct d40_lcla_pool - LCLA pool settings and data. 118 * struct d40_lcla_pool - LCLA pool settings and data.
115 * 119 *
116 * @base: The virtual address of LCLA. 120 * @base: The virtual address of LCLA. 18 bit aligned.
117 * @phy: Physical base address of LCLA. 121 * @base_unaligned: The orignal kmalloc pointer, if kmalloc is used.
118 * @base_size: size of lcla. 122 * This pointer is only there for clean-up on error.
123 * @pages: The number of pages needed for all physical channels.
124 * Only used later for clean-up on error
119 * @lock: Lock to protect the content in this struct. 125 * @lock: Lock to protect the content in this struct.
120 * @alloc_map: Mapping between physical channel and LCLA entries. 126 * @alloc_map: Bitmap mapping between physical channel and LCLA entries.
121 * @num_blocks: The number of entries of alloc_map. Equals to the 127 * @num_blocks: The number of entries of alloc_map. Equals to the
122 * number of physical channels. 128 * number of physical channels.
123 */ 129 */
124struct d40_lcla_pool { 130struct d40_lcla_pool {
125 void *base; 131 void *base;
126 dma_addr_t phy; 132 void *base_unaligned;
127 resource_size_t base_size; 133 int pages;
128 spinlock_t lock; 134 spinlock_t lock;
129 u32 *alloc_map; 135 u32 *alloc_map;
130 int num_blocks; 136 int num_blocks;
@@ -163,15 +169,14 @@ struct d40_base;
163 * @pending_tx: The number of pending transfers. Used between interrupt handler 169 * @pending_tx: The number of pending transfers. Used between interrupt handler
164 * and tasklet. 170 * and tasklet.
165 * @busy: Set to true when transfer is ongoing on this channel. 171 * @busy: Set to true when transfer is ongoing on this channel.
166 * @phy_chan: Pointer to physical channel which this instance runs on. 172 * @phy_chan: Pointer to physical channel which this instance runs on. If this
173 * point is NULL, then the channel is not allocated.
167 * @chan: DMA engine handle. 174 * @chan: DMA engine handle.
168 * @tasklet: Tasklet that gets scheduled from interrupt context to complete a 175 * @tasklet: Tasklet that gets scheduled from interrupt context to complete a
169 * transfer and call client callback. 176 * transfer and call client callback.
170 * @client: Cliented owned descriptor list. 177 * @client: Cliented owned descriptor list.
171 * @active: Active descriptor. 178 * @active: Active descriptor.
172 * @queue: Queued jobs. 179 * @queue: Queued jobs.
173 * @free: List of free descripts, ready to be reused.
174 * @free_len: Number of descriptors in the free list.
175 * @dma_cfg: The client configuration of this dma channel. 180 * @dma_cfg: The client configuration of this dma channel.
176 * @base: Pointer to the device instance struct. 181 * @base: Pointer to the device instance struct.
177 * @src_def_cfg: Default cfg register setting for src. 182 * @src_def_cfg: Default cfg register setting for src.
@@ -195,8 +200,6 @@ struct d40_chan {
195 struct list_head client; 200 struct list_head client;
196 struct list_head active; 201 struct list_head active;
197 struct list_head queue; 202 struct list_head queue;
198 struct list_head free;
199 int free_len;
200 struct stedma40_chan_cfg dma_cfg; 203 struct stedma40_chan_cfg dma_cfg;
201 struct d40_base *base; 204 struct d40_base *base;
202 /* Default register configurations */ 205 /* Default register configurations */
@@ -205,6 +208,9 @@ struct d40_chan {
205 struct d40_def_lcsp log_def; 208 struct d40_def_lcsp log_def;
206 struct d40_lcla_elem lcla; 209 struct d40_lcla_elem lcla;
207 struct d40_log_lli_full *lcpa; 210 struct d40_log_lli_full *lcpa;
211 /* Runtime reconfiguration */
212 dma_addr_t runtime_addr;
213 enum dma_data_direction runtime_direction;
208}; 214};
209 215
210/** 216/**
@@ -215,6 +221,7 @@ struct d40_chan {
215 * the same physical register. 221 * the same physical register.
216 * @dev: The device structure. 222 * @dev: The device structure.
217 * @virtbase: The virtual base address of the DMA's register. 223 * @virtbase: The virtual base address of the DMA's register.
224 * @rev: silicon revision detected.
218 * @clk: Pointer to the DMA clock structure. 225 * @clk: Pointer to the DMA clock structure.
219 * @phy_start: Physical memory start of the DMA registers. 226 * @phy_start: Physical memory start of the DMA registers.
220 * @phy_size: Size of the DMA register map. 227 * @phy_size: Size of the DMA register map.
@@ -240,12 +247,14 @@ struct d40_chan {
240 * @lcpa_base: The virtual mapped address of LCPA. 247 * @lcpa_base: The virtual mapped address of LCPA.
241 * @phy_lcpa: The physical address of the LCPA. 248 * @phy_lcpa: The physical address of the LCPA.
242 * @lcpa_size: The size of the LCPA area. 249 * @lcpa_size: The size of the LCPA area.
250 * @desc_slab: cache for descriptors.
243 */ 251 */
244struct d40_base { 252struct d40_base {
245 spinlock_t interrupt_lock; 253 spinlock_t interrupt_lock;
246 spinlock_t execmd_lock; 254 spinlock_t execmd_lock;
247 struct device *dev; 255 struct device *dev;
248 void __iomem *virtbase; 256 void __iomem *virtbase;
257 u8 rev:4;
249 struct clk *clk; 258 struct clk *clk;
250 phys_addr_t phy_start; 259 phys_addr_t phy_start;
251 resource_size_t phy_size; 260 resource_size_t phy_size;
@@ -266,6 +275,7 @@ struct d40_base {
266 void *lcpa_base; 275 void *lcpa_base;
267 dma_addr_t phy_lcpa; 276 dma_addr_t phy_lcpa;
268 resource_size_t lcpa_size; 277 resource_size_t lcpa_size;
278 struct kmem_cache *desc_slab;
269}; 279};
270 280
271/** 281/**
@@ -365,11 +375,6 @@ static dma_cookie_t d40_assign_cookie(struct d40_chan *d40c,
365 return cookie; 375 return cookie;
366} 376}
367 377
368static void d40_desc_reset(struct d40_desc *d40d)
369{
370 d40d->lli_tcount = 0;
371}
372
373static void d40_desc_remove(struct d40_desc *d40d) 378static void d40_desc_remove(struct d40_desc *d40d)
374{ 379{
375 list_del(&d40d->node); 380 list_del(&d40d->node);
@@ -377,7 +382,6 @@ static void d40_desc_remove(struct d40_desc *d40d)
377 382
378static struct d40_desc *d40_desc_get(struct d40_chan *d40c) 383static struct d40_desc *d40_desc_get(struct d40_chan *d40c)
379{ 384{
380 struct d40_desc *desc;
381 struct d40_desc *d; 385 struct d40_desc *d;
382 struct d40_desc *_d; 386 struct d40_desc *_d;
383 387
@@ -386,36 +390,21 @@ static struct d40_desc *d40_desc_get(struct d40_chan *d40c)
386 if (async_tx_test_ack(&d->txd)) { 390 if (async_tx_test_ack(&d->txd)) {
387 d40_pool_lli_free(d); 391 d40_pool_lli_free(d);
388 d40_desc_remove(d); 392 d40_desc_remove(d);
389 desc = d; 393 break;
390 goto out;
391 } 394 }
392 }
393
394 if (list_empty(&d40c->free)) {
395 /* Alloc new desc because we're out of used ones */
396 desc = kzalloc(sizeof(struct d40_desc), GFP_NOWAIT);
397 if (desc == NULL)
398 goto out;
399 INIT_LIST_HEAD(&desc->node);
400 } else { 395 } else {
401 /* Reuse an old desc. */ 396 d = kmem_cache_alloc(d40c->base->desc_slab, GFP_NOWAIT);
402 desc = list_first_entry(&d40c->free, 397 if (d != NULL) {
403 struct d40_desc, 398 memset(d, 0, sizeof(struct d40_desc));
404 node); 399 INIT_LIST_HEAD(&d->node);
405 list_del(&desc->node); 400 }
406 d40c->free_len--;
407 } 401 }
408out: 402 return d;
409 return desc;
410} 403}
411 404
412static void d40_desc_free(struct d40_chan *d40c, struct d40_desc *d40d) 405static void d40_desc_free(struct d40_chan *d40c, struct d40_desc *d40d)
413{ 406{
414 if (d40c->free_len < D40_DESC_CACHE_SIZE) { 407 kmem_cache_free(d40c->base->desc_slab, d40d);
415 list_add_tail(&d40d->node, &d40c->free);
416 d40c->free_len++;
417 } else
418 kfree(d40d);
419} 408}
420 409
421static void d40_desc_submit(struct d40_chan *d40c, struct d40_desc *desc) 410static void d40_desc_submit(struct d40_chan *d40c, struct d40_desc *desc)
@@ -456,37 +445,41 @@ static struct d40_desc *d40_first_queued(struct d40_chan *d40c)
456 445
457/* Support functions for logical channels */ 446/* Support functions for logical channels */
458 447
459static int d40_lcla_id_get(struct d40_chan *d40c, 448static int d40_lcla_id_get(struct d40_chan *d40c)
460 struct d40_lcla_pool *pool)
461{ 449{
462 int src_id = 0; 450 int src_id = 0;
463 int dst_id = 0; 451 int dst_id = 0;
464 struct d40_log_lli *lcla_lidx_base = 452 struct d40_log_lli *lcla_lidx_base =
465 pool->base + d40c->phy_chan->num * 1024; 453 d40c->base->lcla_pool.base + d40c->phy_chan->num * 1024;
466 int i; 454 int i;
467 int lli_per_log = d40c->base->plat_data->llis_per_log; 455 int lli_per_log = d40c->base->plat_data->llis_per_log;
456 unsigned long flags;
468 457
469 if (d40c->lcla.src_id >= 0 && d40c->lcla.dst_id >= 0) 458 if (d40c->lcla.src_id >= 0 && d40c->lcla.dst_id >= 0)
470 return 0; 459 return 0;
471 460
472 if (pool->num_blocks > 32) 461 if (d40c->base->lcla_pool.num_blocks > 32)
473 return -EINVAL; 462 return -EINVAL;
474 463
475 spin_lock(&pool->lock); 464 spin_lock_irqsave(&d40c->base->lcla_pool.lock, flags);
476 465
477 for (i = 0; i < pool->num_blocks; i++) { 466 for (i = 0; i < d40c->base->lcla_pool.num_blocks; i++) {
478 if (!(pool->alloc_map[d40c->phy_chan->num] & (0x1 << i))) { 467 if (!(d40c->base->lcla_pool.alloc_map[d40c->phy_chan->num] &
479 pool->alloc_map[d40c->phy_chan->num] |= (0x1 << i); 468 (0x1 << i))) {
469 d40c->base->lcla_pool.alloc_map[d40c->phy_chan->num] |=
470 (0x1 << i);
480 break; 471 break;
481 } 472 }
482 } 473 }
483 src_id = i; 474 src_id = i;
484 if (src_id >= pool->num_blocks) 475 if (src_id >= d40c->base->lcla_pool.num_blocks)
485 goto err; 476 goto err;
486 477
487 for (; i < pool->num_blocks; i++) { 478 for (; i < d40c->base->lcla_pool.num_blocks; i++) {
488 if (!(pool->alloc_map[d40c->phy_chan->num] & (0x1 << i))) { 479 if (!(d40c->base->lcla_pool.alloc_map[d40c->phy_chan->num] &
489 pool->alloc_map[d40c->phy_chan->num] |= (0x1 << i); 480 (0x1 << i))) {
481 d40c->base->lcla_pool.alloc_map[d40c->phy_chan->num] |=
482 (0x1 << i);
490 break; 483 break;
491 } 484 }
492 } 485 }
@@ -500,28 +493,13 @@ static int d40_lcla_id_get(struct d40_chan *d40c,
500 d40c->lcla.dst = lcla_lidx_base + dst_id * lli_per_log + 1; 493 d40c->lcla.dst = lcla_lidx_base + dst_id * lli_per_log + 1;
501 d40c->lcla.src = lcla_lidx_base + src_id * lli_per_log + 1; 494 d40c->lcla.src = lcla_lidx_base + src_id * lli_per_log + 1;
502 495
503 496 spin_unlock_irqrestore(&d40c->base->lcla_pool.lock, flags);
504 spin_unlock(&pool->lock);
505 return 0; 497 return 0;
506err: 498err:
507 spin_unlock(&pool->lock); 499 spin_unlock_irqrestore(&d40c->base->lcla_pool.lock, flags);
508 return -EINVAL; 500 return -EINVAL;
509} 501}
510 502
511static void d40_lcla_id_put(struct d40_chan *d40c,
512 struct d40_lcla_pool *pool,
513 int id)
514{
515 if (id < 0)
516 return;
517
518 d40c->lcla.src_id = -1;
519 d40c->lcla.dst_id = -1;
520
521 spin_lock(&pool->lock);
522 pool->alloc_map[d40c->phy_chan->num] &= (~(0x1 << id));
523 spin_unlock(&pool->lock);
524}
525 503
526static int d40_channel_execute_command(struct d40_chan *d40c, 504static int d40_channel_execute_command(struct d40_chan *d40c,
527 enum d40_command command) 505 enum d40_command command)
@@ -530,6 +508,7 @@ static int d40_channel_execute_command(struct d40_chan *d40c,
530 void __iomem *active_reg; 508 void __iomem *active_reg;
531 int ret = 0; 509 int ret = 0;
532 unsigned long flags; 510 unsigned long flags;
511 u32 wmask;
533 512
534 spin_lock_irqsave(&d40c->base->execmd_lock, flags); 513 spin_lock_irqsave(&d40c->base->execmd_lock, flags);
535 514
@@ -547,7 +526,9 @@ static int d40_channel_execute_command(struct d40_chan *d40c,
547 goto done; 526 goto done;
548 } 527 }
549 528
550 writel(command << D40_CHAN_POS(d40c->phy_chan->num), active_reg); 529 wmask = 0xffffffff & ~(D40_CHAN_POS_MASK(d40c->phy_chan->num));
530 writel(wmask | (command << D40_CHAN_POS(d40c->phy_chan->num)),
531 active_reg);
551 532
552 if (command == D40_DMA_SUSPEND_REQ) { 533 if (command == D40_DMA_SUSPEND_REQ) {
553 534
@@ -586,8 +567,7 @@ done:
586static void d40_term_all(struct d40_chan *d40c) 567static void d40_term_all(struct d40_chan *d40c)
587{ 568{
588 struct d40_desc *d40d; 569 struct d40_desc *d40d;
589 struct d40_desc *d; 570 unsigned long flags;
590 struct d40_desc *_d;
591 571
592 /* Release active descriptors */ 572 /* Release active descriptors */
593 while ((d40d = d40_first_active_get(d40c))) { 573 while ((d40d = d40_first_active_get(d40c))) {
@@ -605,19 +585,17 @@ static void d40_term_all(struct d40_chan *d40c)
605 d40_desc_free(d40c, d40d); 585 d40_desc_free(d40c, d40d);
606 } 586 }
607 587
608 /* Release client owned descriptors */ 588 spin_lock_irqsave(&d40c->base->lcla_pool.lock, flags);
609 if (!list_empty(&d40c->client)) 589
610 list_for_each_entry_safe(d, _d, &d40c->client, node) { 590 d40c->base->lcla_pool.alloc_map[d40c->phy_chan->num] &=
611 d40_pool_lli_free(d); 591 (~(0x1 << d40c->lcla.dst_id));
612 d40_desc_remove(d); 592 d40c->base->lcla_pool.alloc_map[d40c->phy_chan->num] &=
613 /* Return desc to free-list */ 593 (~(0x1 << d40c->lcla.src_id));
614 d40_desc_free(d40c, d40d); 594
615 } 595 d40c->lcla.src_id = -1;
596 d40c->lcla.dst_id = -1;
616 597
617 d40_lcla_id_put(d40c, &d40c->base->lcla_pool, 598 spin_unlock_irqrestore(&d40c->base->lcla_pool.lock, flags);
618 d40c->lcla.src_id);
619 d40_lcla_id_put(d40c, &d40c->base->lcla_pool,
620 d40c->lcla.dst_id);
621 599
622 d40c->pending_tx = 0; 600 d40c->pending_tx = 0;
623 d40c->busy = false; 601 d40c->busy = false;
@@ -628,6 +606,7 @@ static void d40_config_set_event(struct d40_chan *d40c, bool do_enable)
628 u32 val; 606 u32 val;
629 unsigned long flags; 607 unsigned long flags;
630 608
609 /* Notice, that disable requires the physical channel to be stopped */
631 if (do_enable) 610 if (do_enable)
632 val = D40_ACTIVATE_EVENTLINE; 611 val = D40_ACTIVATE_EVENTLINE;
633 else 612 else
@@ -732,31 +711,34 @@ static int d40_config_write(struct d40_chan *d40c)
732 711
733static void d40_desc_load(struct d40_chan *d40c, struct d40_desc *d40d) 712static void d40_desc_load(struct d40_chan *d40c, struct d40_desc *d40d)
734{ 713{
735
736 if (d40d->lli_phy.dst && d40d->lli_phy.src) { 714 if (d40d->lli_phy.dst && d40d->lli_phy.src) {
737 d40_phy_lli_write(d40c->base->virtbase, 715 d40_phy_lli_write(d40c->base->virtbase,
738 d40c->phy_chan->num, 716 d40c->phy_chan->num,
739 d40d->lli_phy.dst, 717 d40d->lli_phy.dst,
740 d40d->lli_phy.src); 718 d40d->lli_phy.src);
741 d40d->lli_tcount = d40d->lli_len;
742 } else if (d40d->lli_log.dst && d40d->lli_log.src) { 719 } else if (d40d->lli_log.dst && d40d->lli_log.src) {
743 u32 lli_len;
744 struct d40_log_lli *src = d40d->lli_log.src; 720 struct d40_log_lli *src = d40d->lli_log.src;
745 struct d40_log_lli *dst = d40d->lli_log.dst; 721 struct d40_log_lli *dst = d40d->lli_log.dst;
746 722 int s;
747 src += d40d->lli_tcount; 723
748 dst += d40d->lli_tcount; 724 src += d40d->lli_count;
749 725 dst += d40d->lli_count;
750 if (d40d->lli_len <= d40c->base->plat_data->llis_per_log) 726 s = d40_log_lli_write(d40c->lcpa,
751 lli_len = d40d->lli_len; 727 d40c->lcla.src, d40c->lcla.dst,
752 else 728 dst, src,
753 lli_len = d40c->base->plat_data->llis_per_log; 729 d40c->base->plat_data->llis_per_log);
754 d40d->lli_tcount += lli_len; 730
755 d40_log_lli_write(d40c->lcpa, d40c->lcla.src, 731 /* If s equals to zero, the job is not linked */
756 d40c->lcla.dst, 732 if (s > 0) {
757 dst, src, 733 (void) dma_map_single(d40c->base->dev, d40c->lcla.src,
758 d40c->base->plat_data->llis_per_log); 734 s * sizeof(struct d40_log_lli),
735 DMA_TO_DEVICE);
736 (void) dma_map_single(d40c->base->dev, d40c->lcla.dst,
737 s * sizeof(struct d40_log_lli),
738 DMA_TO_DEVICE);
739 }
759 } 740 }
741 d40d->lli_count += d40d->lli_tx_len;
760} 742}
761 743
762static dma_cookie_t d40_tx_submit(struct dma_async_tx_descriptor *tx) 744static dma_cookie_t d40_tx_submit(struct dma_async_tx_descriptor *tx)
@@ -780,18 +762,21 @@ static dma_cookie_t d40_tx_submit(struct dma_async_tx_descriptor *tx)
780 762
781static int d40_start(struct d40_chan *d40c) 763static int d40_start(struct d40_chan *d40c)
782{ 764{
783 int err; 765 if (d40c->base->rev == 0) {
766 int err;
784 767
785 if (d40c->log_num != D40_PHY_CHAN) { 768 if (d40c->log_num != D40_PHY_CHAN) {
786 err = d40_channel_execute_command(d40c, D40_DMA_SUSPEND_REQ); 769 err = d40_channel_execute_command(d40c,
787 if (err) 770 D40_DMA_SUSPEND_REQ);
788 return err; 771 if (err)
789 d40_config_set_event(d40c, true); 772 return err;
773 }
790 } 774 }
791 775
792 err = d40_channel_execute_command(d40c, D40_DMA_RUN); 776 if (d40c->log_num != D40_PHY_CHAN)
777 d40_config_set_event(d40c, true);
793 778
794 return err; 779 return d40_channel_execute_command(d40c, D40_DMA_RUN);
795} 780}
796 781
797static struct d40_desc *d40_queue_start(struct d40_chan *d40c) 782static struct d40_desc *d40_queue_start(struct d40_chan *d40c)
@@ -838,7 +823,7 @@ static void dma_tc_handle(struct d40_chan *d40c)
838 if (d40d == NULL) 823 if (d40d == NULL)
839 return; 824 return;
840 825
841 if (d40d->lli_tcount < d40d->lli_len) { 826 if (d40d->lli_count < d40d->lli_len) {
842 827
843 d40_desc_load(d40c, d40d); 828 d40_desc_load(d40c, d40d);
844 /* Start dma job */ 829 /* Start dma job */
@@ -891,7 +876,6 @@ static void dma_tasklet(unsigned long data)
891 /* Return desc to free-list */ 876 /* Return desc to free-list */
892 d40_desc_free(d40c, d40d_fin); 877 d40_desc_free(d40c, d40d_fin);
893 } else { 878 } else {
894 d40_desc_reset(d40d_fin);
895 if (!d40d_fin->is_in_client_list) { 879 if (!d40d_fin->is_in_client_list) {
896 d40_desc_remove(d40d_fin); 880 d40_desc_remove(d40d_fin);
897 list_add_tail(&d40d_fin->node, &d40c->client); 881 list_add_tail(&d40d_fin->node, &d40c->client);
@@ -975,7 +959,8 @@ static irqreturn_t d40_handle_interrupt(int irq, void *data)
975 if (!il[row].is_error) 959 if (!il[row].is_error)
976 dma_tc_handle(d40c); 960 dma_tc_handle(d40c);
977 else 961 else
978 dev_err(base->dev, "[%s] IRQ chan: %ld offset %d idx %d\n", 962 dev_err(base->dev,
963 "[%s] IRQ chan: %ld offset %d idx %d\n",
979 __func__, chan, il[row].offset, idx); 964 __func__, chan, il[row].offset, idx);
980 965
981 spin_unlock(&d40c->lock); 966 spin_unlock(&d40c->lock);
@@ -1134,7 +1119,8 @@ static int d40_allocate_channel(struct d40_chan *d40c)
1134 int j; 1119 int j;
1135 int log_num; 1120 int log_num;
1136 bool is_src; 1121 bool is_src;
1137 bool is_log = (d40c->dma_cfg.channel_type & STEDMA40_CHANNEL_IN_OPER_MODE) 1122 bool is_log = (d40c->dma_cfg.channel_type &
1123 STEDMA40_CHANNEL_IN_OPER_MODE)
1138 == STEDMA40_CHANNEL_IN_LOG_MODE; 1124 == STEDMA40_CHANNEL_IN_LOG_MODE;
1139 1125
1140 1126
@@ -1169,8 +1155,10 @@ static int d40_allocate_channel(struct d40_chan *d40c)
1169 for (j = 0; j < d40c->base->num_phy_chans; j += 8) { 1155 for (j = 0; j < d40c->base->num_phy_chans; j += 8) {
1170 int phy_num = j + event_group * 2; 1156 int phy_num = j + event_group * 2;
1171 for (i = phy_num; i < phy_num + 2; i++) { 1157 for (i = phy_num; i < phy_num + 2; i++) {
1172 if (d40_alloc_mask_set(&phys[i], is_src, 1158 if (d40_alloc_mask_set(&phys[i],
1173 0, is_log)) 1159 is_src,
1160 0,
1161 is_log))
1174 goto found_phy; 1162 goto found_phy;
1175 } 1163 }
1176 } 1164 }
@@ -1221,30 +1209,6 @@ out:
1221 1209
1222} 1210}
1223 1211
1224static int d40_config_chan(struct d40_chan *d40c,
1225 struct stedma40_chan_cfg *info)
1226{
1227
1228 /* Fill in basic CFG register values */
1229 d40_phy_cfg(&d40c->dma_cfg, &d40c->src_def_cfg,
1230 &d40c->dst_def_cfg, d40c->log_num != D40_PHY_CHAN);
1231
1232 if (d40c->log_num != D40_PHY_CHAN) {
1233 d40_log_cfg(&d40c->dma_cfg,
1234 &d40c->log_def.lcsp1, &d40c->log_def.lcsp3);
1235
1236 if (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM)
1237 d40c->lcpa = d40c->base->lcpa_base +
1238 d40c->dma_cfg.src_dev_type * 32;
1239 else
1240 d40c->lcpa = d40c->base->lcpa_base +
1241 d40c->dma_cfg.dst_dev_type * 32 + 16;
1242 }
1243
1244 /* Write channel configuration to the DMA */
1245 return d40_config_write(d40c);
1246}
1247
1248static int d40_config_memcpy(struct d40_chan *d40c) 1212static int d40_config_memcpy(struct d40_chan *d40c)
1249{ 1213{
1250 dma_cap_mask_t cap = d40c->chan.device->cap_mask; 1214 dma_cap_mask_t cap = d40c->chan.device->cap_mask;
@@ -1272,13 +1236,25 @@ static int d40_free_dma(struct d40_chan *d40c)
1272{ 1236{
1273 1237
1274 int res = 0; 1238 int res = 0;
1275 u32 event, dir; 1239 u32 event;
1276 struct d40_phy_res *phy = d40c->phy_chan; 1240 struct d40_phy_res *phy = d40c->phy_chan;
1277 bool is_src; 1241 bool is_src;
1242 struct d40_desc *d;
1243 struct d40_desc *_d;
1244
1278 1245
1279 /* Terminate all queued and active transfers */ 1246 /* Terminate all queued and active transfers */
1280 d40_term_all(d40c); 1247 d40_term_all(d40c);
1281 1248
1249 /* Release client owned descriptors */
1250 if (!list_empty(&d40c->client))
1251 list_for_each_entry_safe(d, _d, &d40c->client, node) {
1252 d40_pool_lli_free(d);
1253 d40_desc_remove(d);
1254 /* Return desc to free-list */
1255 d40_desc_free(d40c, d);
1256 }
1257
1282 if (phy == NULL) { 1258 if (phy == NULL) {
1283 dev_err(&d40c->chan.dev->device, "[%s] phy == null\n", 1259 dev_err(&d40c->chan.dev->device, "[%s] phy == null\n",
1284 __func__); 1260 __func__);
@@ -1292,22 +1268,12 @@ static int d40_free_dma(struct d40_chan *d40c)
1292 return -EINVAL; 1268 return -EINVAL;
1293 } 1269 }
1294 1270
1295
1296 res = d40_channel_execute_command(d40c, D40_DMA_SUSPEND_REQ);
1297 if (res) {
1298 dev_err(&d40c->chan.dev->device, "[%s] suspend\n",
1299 __func__);
1300 return res;
1301 }
1302
1303 if (d40c->dma_cfg.dir == STEDMA40_MEM_TO_PERIPH || 1271 if (d40c->dma_cfg.dir == STEDMA40_MEM_TO_PERIPH ||
1304 d40c->dma_cfg.dir == STEDMA40_MEM_TO_MEM) { 1272 d40c->dma_cfg.dir == STEDMA40_MEM_TO_MEM) {
1305 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.dst_dev_type); 1273 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.dst_dev_type);
1306 dir = D40_CHAN_REG_SDLNK;
1307 is_src = false; 1274 is_src = false;
1308 } else if (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM) { 1275 } else if (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM) {
1309 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.src_dev_type); 1276 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.src_dev_type);
1310 dir = D40_CHAN_REG_SSLNK;
1311 is_src = true; 1277 is_src = true;
1312 } else { 1278 } else {
1313 dev_err(&d40c->chan.dev->device, 1279 dev_err(&d40c->chan.dev->device,
@@ -1315,16 +1281,17 @@ static int d40_free_dma(struct d40_chan *d40c)
1315 return -EINVAL; 1281 return -EINVAL;
1316 } 1282 }
1317 1283
1284 res = d40_channel_execute_command(d40c, D40_DMA_SUSPEND_REQ);
1285 if (res) {
1286 dev_err(&d40c->chan.dev->device, "[%s] suspend failed\n",
1287 __func__);
1288 return res;
1289 }
1290
1318 if (d40c->log_num != D40_PHY_CHAN) { 1291 if (d40c->log_num != D40_PHY_CHAN) {
1319 /* 1292 /* Release logical channel, deactivate the event line */
1320 * Release logical channel, deactivate the event line during
1321 * the time physical res is suspended.
1322 */
1323 writel((D40_DEACTIVATE_EVENTLINE << D40_EVENTLINE_POS(event)) &
1324 D40_EVENTLINE_MASK(event),
1325 d40c->base->virtbase + D40_DREG_PCBASE +
1326 phy->num * D40_DREG_PCDELTA + dir);
1327 1293
1294 d40_config_set_event(d40c, false);
1328 d40c->base->lookup_log_chans[d40c->log_num] = NULL; 1295 d40c->base->lookup_log_chans[d40c->log_num] = NULL;
1329 1296
1330 /* 1297 /*
@@ -1345,8 +1312,9 @@ static int d40_free_dma(struct d40_chan *d40c)
1345 } 1312 }
1346 return 0; 1313 return 0;
1347 } 1314 }
1348 } else 1315 } else {
1349 d40_alloc_mask_free(phy, is_src, 0); 1316 (void) d40_alloc_mask_free(phy, is_src, 0);
1317 }
1350 1318
1351 /* Release physical channel */ 1319 /* Release physical channel */
1352 res = d40_channel_execute_command(d40c, D40_DMA_STOP); 1320 res = d40_channel_execute_command(d40c, D40_DMA_STOP);
@@ -1361,8 +1329,6 @@ static int d40_free_dma(struct d40_chan *d40c)
1361 d40c->base->lookup_phy_chans[phy->num] = NULL; 1329 d40c->base->lookup_phy_chans[phy->num] = NULL;
1362 1330
1363 return 0; 1331 return 0;
1364
1365
1366} 1332}
1367 1333
1368static int d40_pause(struct dma_chan *chan) 1334static int d40_pause(struct dma_chan *chan)
@@ -1370,7 +1336,6 @@ static int d40_pause(struct dma_chan *chan)
1370 struct d40_chan *d40c = 1336 struct d40_chan *d40c =
1371 container_of(chan, struct d40_chan, chan); 1337 container_of(chan, struct d40_chan, chan);
1372 int res; 1338 int res;
1373
1374 unsigned long flags; 1339 unsigned long flags;
1375 1340
1376 spin_lock_irqsave(&d40c->lock, flags); 1341 spin_lock_irqsave(&d40c->lock, flags);
@@ -1397,7 +1362,6 @@ static bool d40_is_paused(struct d40_chan *d40c)
1397 void __iomem *active_reg; 1362 void __iomem *active_reg;
1398 u32 status; 1363 u32 status;
1399 u32 event; 1364 u32 event;
1400 int res;
1401 1365
1402 spin_lock_irqsave(&d40c->lock, flags); 1366 spin_lock_irqsave(&d40c->lock, flags);
1403 1367
@@ -1416,10 +1380,6 @@ static bool d40_is_paused(struct d40_chan *d40c)
1416 goto _exit; 1380 goto _exit;
1417 } 1381 }
1418 1382
1419 res = d40_channel_execute_command(d40c, D40_DMA_SUSPEND_REQ);
1420 if (res != 0)
1421 goto _exit;
1422
1423 if (d40c->dma_cfg.dir == STEDMA40_MEM_TO_PERIPH || 1383 if (d40c->dma_cfg.dir == STEDMA40_MEM_TO_PERIPH ||
1424 d40c->dma_cfg.dir == STEDMA40_MEM_TO_MEM) 1384 d40c->dma_cfg.dir == STEDMA40_MEM_TO_MEM)
1425 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.dst_dev_type); 1385 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.dst_dev_type);
@@ -1436,12 +1396,6 @@ static bool d40_is_paused(struct d40_chan *d40c)
1436 1396
1437 if (status != D40_DMA_RUN) 1397 if (status != D40_DMA_RUN)
1438 is_paused = true; 1398 is_paused = true;
1439
1440 /* Resume the other logical channels if any */
1441 if (d40_chan_has_events(d40c))
1442 res = d40_channel_execute_command(d40c,
1443 D40_DMA_RUN);
1444
1445_exit: 1399_exit:
1446 spin_unlock_irqrestore(&d40c->lock, flags); 1400 spin_unlock_irqrestore(&d40c->lock, flags);
1447 return is_paused; 1401 return is_paused;
@@ -1468,13 +1422,14 @@ static u32 d40_residue(struct d40_chan *d40c)
1468 u32 num_elt; 1422 u32 num_elt;
1469 1423
1470 if (d40c->log_num != D40_PHY_CHAN) 1424 if (d40c->log_num != D40_PHY_CHAN)
1471 num_elt = (readl(&d40c->lcpa->lcsp2) & D40_MEM_LCSP2_ECNT_MASK) 1425 num_elt = (readl(&d40c->lcpa->lcsp2) & D40_MEM_LCSP2_ECNT_MASK)
1472 >> D40_MEM_LCSP2_ECNT_POS; 1426 >> D40_MEM_LCSP2_ECNT_POS;
1473 else 1427 else
1474 num_elt = (readl(d40c->base->virtbase + D40_DREG_PCBASE + 1428 num_elt = (readl(d40c->base->virtbase + D40_DREG_PCBASE +
1475 d40c->phy_chan->num * D40_DREG_PCDELTA + 1429 d40c->phy_chan->num * D40_DREG_PCDELTA +
1476 D40_CHAN_REG_SDELT) & 1430 D40_CHAN_REG_SDELT) &
1477 D40_SREG_ELEM_PHY_ECNT_MASK) >> D40_SREG_ELEM_PHY_ECNT_POS; 1431 D40_SREG_ELEM_PHY_ECNT_MASK) >>
1432 D40_SREG_ELEM_PHY_ECNT_POS;
1478 return num_elt * (1 << d40c->dma_cfg.dst_info.data_width); 1433 return num_elt * (1 << d40c->dma_cfg.dst_info.data_width);
1479} 1434}
1480 1435
@@ -1487,20 +1442,21 @@ static int d40_resume(struct dma_chan *chan)
1487 1442
1488 spin_lock_irqsave(&d40c->lock, flags); 1443 spin_lock_irqsave(&d40c->lock, flags);
1489 1444
1490 if (d40c->log_num != D40_PHY_CHAN) { 1445 if (d40c->base->rev == 0)
1491 res = d40_channel_execute_command(d40c, D40_DMA_SUSPEND_REQ); 1446 if (d40c->log_num != D40_PHY_CHAN) {
1492 if (res) 1447 res = d40_channel_execute_command(d40c,
1493 goto out; 1448 D40_DMA_SUSPEND_REQ);
1449 goto no_suspend;
1450 }
1494 1451
1495 /* If bytes left to transfer or linked tx resume job */ 1452 /* If bytes left to transfer or linked tx resume job */
1496 if (d40_residue(d40c) || d40_tx_is_linked(d40c)) { 1453 if (d40_residue(d40c) || d40_tx_is_linked(d40c)) {
1454 if (d40c->log_num != D40_PHY_CHAN)
1497 d40_config_set_event(d40c, true); 1455 d40_config_set_event(d40c, true);
1498 res = d40_channel_execute_command(d40c, D40_DMA_RUN);
1499 }
1500 } else if (d40_residue(d40c) || d40_tx_is_linked(d40c))
1501 res = d40_channel_execute_command(d40c, D40_DMA_RUN); 1456 res = d40_channel_execute_command(d40c, D40_DMA_RUN);
1457 }
1502 1458
1503out: 1459no_suspend:
1504 spin_unlock_irqrestore(&d40c->lock, flags); 1460 spin_unlock_irqrestore(&d40c->lock, flags);
1505 return res; 1461 return res;
1506} 1462}
@@ -1534,8 +1490,10 @@ int stedma40_set_psize(struct dma_chan *chan,
1534 if (d40c->log_num != D40_PHY_CHAN) { 1490 if (d40c->log_num != D40_PHY_CHAN) {
1535 d40c->log_def.lcsp1 &= ~D40_MEM_LCSP1_SCFG_PSIZE_MASK; 1491 d40c->log_def.lcsp1 &= ~D40_MEM_LCSP1_SCFG_PSIZE_MASK;
1536 d40c->log_def.lcsp3 &= ~D40_MEM_LCSP1_SCFG_PSIZE_MASK; 1492 d40c->log_def.lcsp3 &= ~D40_MEM_LCSP1_SCFG_PSIZE_MASK;
1537 d40c->log_def.lcsp1 |= src_psize << D40_MEM_LCSP1_SCFG_PSIZE_POS; 1493 d40c->log_def.lcsp1 |= src_psize <<
1538 d40c->log_def.lcsp3 |= dst_psize << D40_MEM_LCSP1_SCFG_PSIZE_POS; 1494 D40_MEM_LCSP1_SCFG_PSIZE_POS;
1495 d40c->log_def.lcsp3 |= dst_psize <<
1496 D40_MEM_LCSP1_SCFG_PSIZE_POS;
1539 goto out; 1497 goto out;
1540 } 1498 }
1541 1499
@@ -1566,37 +1524,42 @@ struct dma_async_tx_descriptor *stedma40_memcpy_sg(struct dma_chan *chan,
1566 struct scatterlist *sgl_dst, 1524 struct scatterlist *sgl_dst,
1567 struct scatterlist *sgl_src, 1525 struct scatterlist *sgl_src,
1568 unsigned int sgl_len, 1526 unsigned int sgl_len,
1569 unsigned long flags) 1527 unsigned long dma_flags)
1570{ 1528{
1571 int res; 1529 int res;
1572 struct d40_desc *d40d; 1530 struct d40_desc *d40d;
1573 struct d40_chan *d40c = container_of(chan, struct d40_chan, 1531 struct d40_chan *d40c = container_of(chan, struct d40_chan,
1574 chan); 1532 chan);
1575 unsigned long flg; 1533 unsigned long flags;
1576 int lli_max = d40c->base->plat_data->llis_per_log;
1577 1534
1535 if (d40c->phy_chan == NULL) {
1536 dev_err(&d40c->chan.dev->device,
1537 "[%s] Unallocated channel.\n", __func__);
1538 return ERR_PTR(-EINVAL);
1539 }
1578 1540
1579 spin_lock_irqsave(&d40c->lock, flg); 1541 spin_lock_irqsave(&d40c->lock, flags);
1580 d40d = d40_desc_get(d40c); 1542 d40d = d40_desc_get(d40c);
1581 1543
1582 if (d40d == NULL) 1544 if (d40d == NULL)
1583 goto err; 1545 goto err;
1584 1546
1585 memset(d40d, 0, sizeof(struct d40_desc));
1586 d40d->lli_len = sgl_len; 1547 d40d->lli_len = sgl_len;
1587 1548 d40d->lli_tx_len = d40d->lli_len;
1588 d40d->txd.flags = flags; 1549 d40d->txd.flags = dma_flags;
1589 1550
1590 if (d40c->log_num != D40_PHY_CHAN) { 1551 if (d40c->log_num != D40_PHY_CHAN) {
1552 if (d40d->lli_len > d40c->base->plat_data->llis_per_log)
1553 d40d->lli_tx_len = d40c->base->plat_data->llis_per_log;
1554
1591 if (sgl_len > 1) 1555 if (sgl_len > 1)
1592 /* 1556 /*
1593 * Check if there is space available in lcla. If not, 1557 * Check if there is space available in lcla. If not,
1594 * split list into 1-length and run only in lcpa 1558 * split list into 1-length and run only in lcpa
1595 * space. 1559 * space.
1596 */ 1560 */
1597 if (d40_lcla_id_get(d40c, 1561 if (d40_lcla_id_get(d40c) != 0)
1598 &d40c->base->lcla_pool) != 0) 1562 d40d->lli_tx_len = 1;
1599 lli_max = 1;
1600 1563
1601 if (d40_pool_lli_alloc(d40d, sgl_len, true) < 0) { 1564 if (d40_pool_lli_alloc(d40d, sgl_len, true) < 0) {
1602 dev_err(&d40c->chan.dev->device, 1565 dev_err(&d40c->chan.dev->device,
@@ -1610,7 +1573,8 @@ struct dma_async_tx_descriptor *stedma40_memcpy_sg(struct dma_chan *chan,
1610 d40d->lli_log.src, 1573 d40d->lli_log.src,
1611 d40c->log_def.lcsp1, 1574 d40c->log_def.lcsp1,
1612 d40c->dma_cfg.src_info.data_width, 1575 d40c->dma_cfg.src_info.data_width,
1613 flags & DMA_PREP_INTERRUPT, lli_max, 1576 dma_flags & DMA_PREP_INTERRUPT,
1577 d40d->lli_tx_len,
1614 d40c->base->plat_data->llis_per_log); 1578 d40c->base->plat_data->llis_per_log);
1615 1579
1616 (void) d40_log_sg_to_lli(d40c->lcla.dst_id, 1580 (void) d40_log_sg_to_lli(d40c->lcla.dst_id,
@@ -1619,7 +1583,8 @@ struct dma_async_tx_descriptor *stedma40_memcpy_sg(struct dma_chan *chan,
1619 d40d->lli_log.dst, 1583 d40d->lli_log.dst,
1620 d40c->log_def.lcsp3, 1584 d40c->log_def.lcsp3,
1621 d40c->dma_cfg.dst_info.data_width, 1585 d40c->dma_cfg.dst_info.data_width,
1622 flags & DMA_PREP_INTERRUPT, lli_max, 1586 dma_flags & DMA_PREP_INTERRUPT,
1587 d40d->lli_tx_len,
1623 d40c->base->plat_data->llis_per_log); 1588 d40c->base->plat_data->llis_per_log);
1624 1589
1625 1590
@@ -1664,11 +1629,11 @@ struct dma_async_tx_descriptor *stedma40_memcpy_sg(struct dma_chan *chan,
1664 1629
1665 d40d->txd.tx_submit = d40_tx_submit; 1630 d40d->txd.tx_submit = d40_tx_submit;
1666 1631
1667 spin_unlock_irqrestore(&d40c->lock, flg); 1632 spin_unlock_irqrestore(&d40c->lock, flags);
1668 1633
1669 return &d40d->txd; 1634 return &d40d->txd;
1670err: 1635err:
1671 spin_unlock_irqrestore(&d40c->lock, flg); 1636 spin_unlock_irqrestore(&d40c->lock, flags);
1672 return NULL; 1637 return NULL;
1673} 1638}
1674EXPORT_SYMBOL(stedma40_memcpy_sg); 1639EXPORT_SYMBOL(stedma40_memcpy_sg);
@@ -1698,46 +1663,66 @@ static int d40_alloc_chan_resources(struct dma_chan *chan)
1698 unsigned long flags; 1663 unsigned long flags;
1699 struct d40_chan *d40c = 1664 struct d40_chan *d40c =
1700 container_of(chan, struct d40_chan, chan); 1665 container_of(chan, struct d40_chan, chan);
1701 1666 bool is_free_phy;
1702 spin_lock_irqsave(&d40c->lock, flags); 1667 spin_lock_irqsave(&d40c->lock, flags);
1703 1668
1704 d40c->completed = chan->cookie = 1; 1669 d40c->completed = chan->cookie = 1;
1705 1670
1706 /* 1671 /*
1707 * If no dma configuration is set (channel_type == 0) 1672 * If no dma configuration is set (channel_type == 0)
1708 * use default configuration 1673 * use default configuration (memcpy)
1709 */ 1674 */
1710 if (d40c->dma_cfg.channel_type == 0) { 1675 if (d40c->dma_cfg.channel_type == 0) {
1711 err = d40_config_memcpy(d40c); 1676 err = d40_config_memcpy(d40c);
1712 if (err) 1677 if (err) {
1713 goto err_alloc; 1678 dev_err(&d40c->chan.dev->device,
1679 "[%s] Failed to configure memcpy channel\n",
1680 __func__);
1681 goto fail;
1682 }
1714 } 1683 }
1684 is_free_phy = (d40c->phy_chan == NULL);
1715 1685
1716 err = d40_allocate_channel(d40c); 1686 err = d40_allocate_channel(d40c);
1717 if (err) { 1687 if (err) {
1718 dev_err(&d40c->chan.dev->device, 1688 dev_err(&d40c->chan.dev->device,
1719 "[%s] Failed to allocate channel\n", __func__); 1689 "[%s] Failed to allocate channel\n", __func__);
1720 goto err_alloc; 1690 goto fail;
1721 } 1691 }
1722 1692
1723 err = d40_config_chan(d40c, &d40c->dma_cfg); 1693 /* Fill in basic CFG register values */
1724 if (err) { 1694 d40_phy_cfg(&d40c->dma_cfg, &d40c->src_def_cfg,
1725 dev_err(&d40c->chan.dev->device, 1695 &d40c->dst_def_cfg, d40c->log_num != D40_PHY_CHAN);
1726 "[%s] Failed to configure channel\n",
1727 __func__);
1728 goto err_config;
1729 }
1730 1696
1731 spin_unlock_irqrestore(&d40c->lock, flags); 1697 if (d40c->log_num != D40_PHY_CHAN) {
1732 return 0; 1698 d40_log_cfg(&d40c->dma_cfg,
1699 &d40c->log_def.lcsp1, &d40c->log_def.lcsp3);
1733 1700
1734 err_config: 1701 if (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM)
1735 (void) d40_free_dma(d40c); 1702 d40c->lcpa = d40c->base->lcpa_base +
1736 err_alloc: 1703 d40c->dma_cfg.src_dev_type * D40_LCPA_CHAN_SIZE;
1704 else
1705 d40c->lcpa = d40c->base->lcpa_base +
1706 d40c->dma_cfg.dst_dev_type *
1707 D40_LCPA_CHAN_SIZE + D40_LCPA_CHAN_DST_DELTA;
1708 }
1709
1710 /*
1711 * Only write channel configuration to the DMA if the physical
1712 * resource is free. In case of multiple logical channels
1713 * on the same physical resource, only the first write is necessary.
1714 */
1715 if (is_free_phy) {
1716 err = d40_config_write(d40c);
1717 if (err) {
1718 dev_err(&d40c->chan.dev->device,
1719 "[%s] Failed to configure channel\n",
1720 __func__);
1721 }
1722 }
1723fail:
1737 spin_unlock_irqrestore(&d40c->lock, flags); 1724 spin_unlock_irqrestore(&d40c->lock, flags);
1738 dev_err(&d40c->chan.dev->device, 1725 return err;
1739 "[%s] Channel allocation failed\n", __func__);
1740 return -EINVAL;
1741} 1726}
1742 1727
1743static void d40_free_chan_resources(struct dma_chan *chan) 1728static void d40_free_chan_resources(struct dma_chan *chan)
@@ -1747,6 +1732,13 @@ static void d40_free_chan_resources(struct dma_chan *chan)
1747 int err; 1732 int err;
1748 unsigned long flags; 1733 unsigned long flags;
1749 1734
1735 if (d40c->phy_chan == NULL) {
1736 dev_err(&d40c->chan.dev->device,
1737 "[%s] Cannot free unallocated channel\n", __func__);
1738 return;
1739 }
1740
1741
1750 spin_lock_irqsave(&d40c->lock, flags); 1742 spin_lock_irqsave(&d40c->lock, flags);
1751 1743
1752 err = d40_free_dma(d40c); 1744 err = d40_free_dma(d40c);
@@ -1761,15 +1753,21 @@ static struct dma_async_tx_descriptor *d40_prep_memcpy(struct dma_chan *chan,
1761 dma_addr_t dst, 1753 dma_addr_t dst,
1762 dma_addr_t src, 1754 dma_addr_t src,
1763 size_t size, 1755 size_t size,
1764 unsigned long flags) 1756 unsigned long dma_flags)
1765{ 1757{
1766 struct d40_desc *d40d; 1758 struct d40_desc *d40d;
1767 struct d40_chan *d40c = container_of(chan, struct d40_chan, 1759 struct d40_chan *d40c = container_of(chan, struct d40_chan,
1768 chan); 1760 chan);
1769 unsigned long flg; 1761 unsigned long flags;
1770 int err = 0; 1762 int err = 0;
1771 1763
1772 spin_lock_irqsave(&d40c->lock, flg); 1764 if (d40c->phy_chan == NULL) {
1765 dev_err(&d40c->chan.dev->device,
1766 "[%s] Channel is not allocated.\n", __func__);
1767 return ERR_PTR(-EINVAL);
1768 }
1769
1770 spin_lock_irqsave(&d40c->lock, flags);
1773 d40d = d40_desc_get(d40c); 1771 d40d = d40_desc_get(d40c);
1774 1772
1775 if (d40d == NULL) { 1773 if (d40d == NULL) {
@@ -1778,9 +1776,7 @@ static struct dma_async_tx_descriptor *d40_prep_memcpy(struct dma_chan *chan,
1778 goto err; 1776 goto err;
1779 } 1777 }
1780 1778
1781 memset(d40d, 0, sizeof(struct d40_desc)); 1779 d40d->txd.flags = dma_flags;
1782
1783 d40d->txd.flags = flags;
1784 1780
1785 dma_async_tx_descriptor_init(&d40d->txd, chan); 1781 dma_async_tx_descriptor_init(&d40d->txd, chan);
1786 1782
@@ -1794,6 +1790,7 @@ static struct dma_async_tx_descriptor *d40_prep_memcpy(struct dma_chan *chan,
1794 goto err; 1790 goto err;
1795 } 1791 }
1796 d40d->lli_len = 1; 1792 d40d->lli_len = 1;
1793 d40d->lli_tx_len = 1;
1797 1794
1798 d40_log_fill_lli(d40d->lli_log.src, 1795 d40_log_fill_lli(d40d->lli_log.src,
1799 src, 1796 src,
@@ -1801,7 +1798,7 @@ static struct dma_async_tx_descriptor *d40_prep_memcpy(struct dma_chan *chan,
1801 0, 1798 0,
1802 d40c->log_def.lcsp1, 1799 d40c->log_def.lcsp1,
1803 d40c->dma_cfg.src_info.data_width, 1800 d40c->dma_cfg.src_info.data_width,
1804 true, true); 1801 false, true);
1805 1802
1806 d40_log_fill_lli(d40d->lli_log.dst, 1803 d40_log_fill_lli(d40d->lli_log.dst,
1807 dst, 1804 dst,
@@ -1848,7 +1845,7 @@ static struct dma_async_tx_descriptor *d40_prep_memcpy(struct dma_chan *chan,
1848 d40d->lli_pool.size, DMA_TO_DEVICE); 1845 d40d->lli_pool.size, DMA_TO_DEVICE);
1849 } 1846 }
1850 1847
1851 spin_unlock_irqrestore(&d40c->lock, flg); 1848 spin_unlock_irqrestore(&d40c->lock, flags);
1852 return &d40d->txd; 1849 return &d40d->txd;
1853 1850
1854err_fill_lli: 1851err_fill_lli:
@@ -1856,7 +1853,7 @@ err_fill_lli:
1856 "[%s] Failed filling in PHY LLI\n", __func__); 1853 "[%s] Failed filling in PHY LLI\n", __func__);
1857 d40_pool_lli_free(d40d); 1854 d40_pool_lli_free(d40d);
1858err: 1855err:
1859 spin_unlock_irqrestore(&d40c->lock, flg); 1856 spin_unlock_irqrestore(&d40c->lock, flags);
1860 return NULL; 1857 return NULL;
1861} 1858}
1862 1859
@@ -1865,11 +1862,10 @@ static int d40_prep_slave_sg_log(struct d40_desc *d40d,
1865 struct scatterlist *sgl, 1862 struct scatterlist *sgl,
1866 unsigned int sg_len, 1863 unsigned int sg_len,
1867 enum dma_data_direction direction, 1864 enum dma_data_direction direction,
1868 unsigned long flags) 1865 unsigned long dma_flags)
1869{ 1866{
1870 dma_addr_t dev_addr = 0; 1867 dma_addr_t dev_addr = 0;
1871 int total_size; 1868 int total_size;
1872 int lli_max = d40c->base->plat_data->llis_per_log;
1873 1869
1874 if (d40_pool_lli_alloc(d40d, sg_len, true) < 0) { 1870 if (d40_pool_lli_alloc(d40d, sg_len, true) < 0) {
1875 dev_err(&d40c->chan.dev->device, 1871 dev_err(&d40c->chan.dev->device,
@@ -1878,7 +1874,10 @@ static int d40_prep_slave_sg_log(struct d40_desc *d40d,
1878 } 1874 }
1879 1875
1880 d40d->lli_len = sg_len; 1876 d40d->lli_len = sg_len;
1881 d40d->lli_tcount = 0; 1877 if (d40d->lli_len <= d40c->base->plat_data->llis_per_log)
1878 d40d->lli_tx_len = d40d->lli_len;
1879 else
1880 d40d->lli_tx_len = d40c->base->plat_data->llis_per_log;
1882 1881
1883 if (sg_len > 1) 1882 if (sg_len > 1)
1884 /* 1883 /*
@@ -1886,35 +1885,34 @@ static int d40_prep_slave_sg_log(struct d40_desc *d40d,
1886 * If not, split list into 1-length and run only 1885 * If not, split list into 1-length and run only
1887 * in lcpa space. 1886 * in lcpa space.
1888 */ 1887 */
1889 if (d40_lcla_id_get(d40c, &d40c->base->lcla_pool) != 0) 1888 if (d40_lcla_id_get(d40c) != 0)
1890 lli_max = 1; 1889 d40d->lli_tx_len = 1;
1891 1890
1892 if (direction == DMA_FROM_DEVICE) { 1891 if (direction == DMA_FROM_DEVICE)
1893 dev_addr = d40c->base->plat_data->dev_rx[d40c->dma_cfg.src_dev_type]; 1892 if (d40c->runtime_addr)
1894 total_size = d40_log_sg_to_dev(&d40c->lcla, 1893 dev_addr = d40c->runtime_addr;
1895 sgl, sg_len, 1894 else
1896 &d40d->lli_log, 1895 dev_addr = d40c->base->plat_data->dev_rx[d40c->dma_cfg.src_dev_type];
1897 &d40c->log_def, 1896 else if (direction == DMA_TO_DEVICE)
1898 d40c->dma_cfg.src_info.data_width, 1897 if (d40c->runtime_addr)
1899 d40c->dma_cfg.dst_info.data_width, 1898 dev_addr = d40c->runtime_addr;
1900 direction, 1899 else
1901 flags & DMA_PREP_INTERRUPT, 1900 dev_addr = d40c->base->plat_data->dev_tx[d40c->dma_cfg.dst_dev_type];
1902 dev_addr, lli_max, 1901
1903 d40c->base->plat_data->llis_per_log); 1902 else
1904 } else if (direction == DMA_TO_DEVICE) {
1905 dev_addr = d40c->base->plat_data->dev_tx[d40c->dma_cfg.dst_dev_type];
1906 total_size = d40_log_sg_to_dev(&d40c->lcla,
1907 sgl, sg_len,
1908 &d40d->lli_log,
1909 &d40c->log_def,
1910 d40c->dma_cfg.src_info.data_width,
1911 d40c->dma_cfg.dst_info.data_width,
1912 direction,
1913 flags & DMA_PREP_INTERRUPT,
1914 dev_addr, lli_max,
1915 d40c->base->plat_data->llis_per_log);
1916 } else
1917 return -EINVAL; 1903 return -EINVAL;
1904
1905 total_size = d40_log_sg_to_dev(&d40c->lcla,
1906 sgl, sg_len,
1907 &d40d->lli_log,
1908 &d40c->log_def,
1909 d40c->dma_cfg.src_info.data_width,
1910 d40c->dma_cfg.dst_info.data_width,
1911 direction,
1912 dma_flags & DMA_PREP_INTERRUPT,
1913 dev_addr, d40d->lli_tx_len,
1914 d40c->base->plat_data->llis_per_log);
1915
1918 if (total_size < 0) 1916 if (total_size < 0)
1919 return -EINVAL; 1917 return -EINVAL;
1920 1918
@@ -1926,7 +1924,7 @@ static int d40_prep_slave_sg_phy(struct d40_desc *d40d,
1926 struct scatterlist *sgl, 1924 struct scatterlist *sgl,
1927 unsigned int sgl_len, 1925 unsigned int sgl_len,
1928 enum dma_data_direction direction, 1926 enum dma_data_direction direction,
1929 unsigned long flags) 1927 unsigned long dma_flags)
1930{ 1928{
1931 dma_addr_t src_dev_addr; 1929 dma_addr_t src_dev_addr;
1932 dma_addr_t dst_dev_addr; 1930 dma_addr_t dst_dev_addr;
@@ -1939,13 +1937,19 @@ static int d40_prep_slave_sg_phy(struct d40_desc *d40d,
1939 } 1937 }
1940 1938
1941 d40d->lli_len = sgl_len; 1939 d40d->lli_len = sgl_len;
1942 d40d->lli_tcount = 0; 1940 d40d->lli_tx_len = sgl_len;
1943 1941
1944 if (direction == DMA_FROM_DEVICE) { 1942 if (direction == DMA_FROM_DEVICE) {
1945 dst_dev_addr = 0; 1943 dst_dev_addr = 0;
1946 src_dev_addr = d40c->base->plat_data->dev_rx[d40c->dma_cfg.src_dev_type]; 1944 if (d40c->runtime_addr)
1945 src_dev_addr = d40c->runtime_addr;
1946 else
1947 src_dev_addr = d40c->base->plat_data->dev_rx[d40c->dma_cfg.src_dev_type];
1947 } else if (direction == DMA_TO_DEVICE) { 1948 } else if (direction == DMA_TO_DEVICE) {
1948 dst_dev_addr = d40c->base->plat_data->dev_tx[d40c->dma_cfg.dst_dev_type]; 1949 if (d40c->runtime_addr)
1950 dst_dev_addr = d40c->runtime_addr;
1951 else
1952 dst_dev_addr = d40c->base->plat_data->dev_tx[d40c->dma_cfg.dst_dev_type];
1949 src_dev_addr = 0; 1953 src_dev_addr = 0;
1950 } else 1954 } else
1951 return -EINVAL; 1955 return -EINVAL;
@@ -1983,34 +1987,38 @@ static struct dma_async_tx_descriptor *d40_prep_slave_sg(struct dma_chan *chan,
1983 struct scatterlist *sgl, 1987 struct scatterlist *sgl,
1984 unsigned int sg_len, 1988 unsigned int sg_len,
1985 enum dma_data_direction direction, 1989 enum dma_data_direction direction,
1986 unsigned long flags) 1990 unsigned long dma_flags)
1987{ 1991{
1988 struct d40_desc *d40d; 1992 struct d40_desc *d40d;
1989 struct d40_chan *d40c = container_of(chan, struct d40_chan, 1993 struct d40_chan *d40c = container_of(chan, struct d40_chan,
1990 chan); 1994 chan);
1991 unsigned long flg; 1995 unsigned long flags;
1992 int err; 1996 int err;
1993 1997
1998 if (d40c->phy_chan == NULL) {
1999 dev_err(&d40c->chan.dev->device,
2000 "[%s] Cannot prepare unallocated channel\n", __func__);
2001 return ERR_PTR(-EINVAL);
2002 }
2003
1994 if (d40c->dma_cfg.pre_transfer) 2004 if (d40c->dma_cfg.pre_transfer)
1995 d40c->dma_cfg.pre_transfer(chan, 2005 d40c->dma_cfg.pre_transfer(chan,
1996 d40c->dma_cfg.pre_transfer_data, 2006 d40c->dma_cfg.pre_transfer_data,
1997 sg_dma_len(sgl)); 2007 sg_dma_len(sgl));
1998 2008
1999 spin_lock_irqsave(&d40c->lock, flg); 2009 spin_lock_irqsave(&d40c->lock, flags);
2000 d40d = d40_desc_get(d40c); 2010 d40d = d40_desc_get(d40c);
2001 spin_unlock_irqrestore(&d40c->lock, flg); 2011 spin_unlock_irqrestore(&d40c->lock, flags);
2002 2012
2003 if (d40d == NULL) 2013 if (d40d == NULL)
2004 return NULL; 2014 return NULL;
2005 2015
2006 memset(d40d, 0, sizeof(struct d40_desc));
2007
2008 if (d40c->log_num != D40_PHY_CHAN) 2016 if (d40c->log_num != D40_PHY_CHAN)
2009 err = d40_prep_slave_sg_log(d40d, d40c, sgl, sg_len, 2017 err = d40_prep_slave_sg_log(d40d, d40c, sgl, sg_len,
2010 direction, flags); 2018 direction, dma_flags);
2011 else 2019 else
2012 err = d40_prep_slave_sg_phy(d40d, d40c, sgl, sg_len, 2020 err = d40_prep_slave_sg_phy(d40d, d40c, sgl, sg_len,
2013 direction, flags); 2021 direction, dma_flags);
2014 if (err) { 2022 if (err) {
2015 dev_err(&d40c->chan.dev->device, 2023 dev_err(&d40c->chan.dev->device,
2016 "[%s] Failed to prepare %s slave sg job: %d\n", 2024 "[%s] Failed to prepare %s slave sg job: %d\n",
@@ -2019,7 +2027,7 @@ static struct dma_async_tx_descriptor *d40_prep_slave_sg(struct dma_chan *chan,
2019 return NULL; 2027 return NULL;
2020 } 2028 }
2021 2029
2022 d40d->txd.flags = flags; 2030 d40d->txd.flags = dma_flags;
2023 2031
2024 dma_async_tx_descriptor_init(&d40d->txd, chan); 2032 dma_async_tx_descriptor_init(&d40d->txd, chan);
2025 2033
@@ -2037,6 +2045,13 @@ static enum dma_status d40_tx_status(struct dma_chan *chan,
2037 dma_cookie_t last_complete; 2045 dma_cookie_t last_complete;
2038 int ret; 2046 int ret;
2039 2047
2048 if (d40c->phy_chan == NULL) {
2049 dev_err(&d40c->chan.dev->device,
2050 "[%s] Cannot read status of unallocated channel\n",
2051 __func__);
2052 return -EINVAL;
2053 }
2054
2040 last_complete = d40c->completed; 2055 last_complete = d40c->completed;
2041 last_used = chan->cookie; 2056 last_used = chan->cookie;
2042 2057
@@ -2056,6 +2071,12 @@ static void d40_issue_pending(struct dma_chan *chan)
2056 struct d40_chan *d40c = container_of(chan, struct d40_chan, chan); 2071 struct d40_chan *d40c = container_of(chan, struct d40_chan, chan);
2057 unsigned long flags; 2072 unsigned long flags;
2058 2073
2074 if (d40c->phy_chan == NULL) {
2075 dev_err(&d40c->chan.dev->device,
2076 "[%s] Channel is not allocated!\n", __func__);
2077 return;
2078 }
2079
2059 spin_lock_irqsave(&d40c->lock, flags); 2080 spin_lock_irqsave(&d40c->lock, flags);
2060 2081
2061 /* Busy means that pending jobs are already being processed */ 2082 /* Busy means that pending jobs are already being processed */
@@ -2065,12 +2086,129 @@ static void d40_issue_pending(struct dma_chan *chan)
2065 spin_unlock_irqrestore(&d40c->lock, flags); 2086 spin_unlock_irqrestore(&d40c->lock, flags);
2066} 2087}
2067 2088
2089/* Runtime reconfiguration extension */
2090static void d40_set_runtime_config(struct dma_chan *chan,
2091 struct dma_slave_config *config)
2092{
2093 struct d40_chan *d40c = container_of(chan, struct d40_chan, chan);
2094 struct stedma40_chan_cfg *cfg = &d40c->dma_cfg;
2095 enum dma_slave_buswidth config_addr_width;
2096 dma_addr_t config_addr;
2097 u32 config_maxburst;
2098 enum stedma40_periph_data_width addr_width;
2099 int psize;
2100
2101 if (config->direction == DMA_FROM_DEVICE) {
2102 dma_addr_t dev_addr_rx =
2103 d40c->base->plat_data->dev_rx[cfg->src_dev_type];
2104
2105 config_addr = config->src_addr;
2106 if (dev_addr_rx)
2107 dev_dbg(d40c->base->dev,
2108 "channel has a pre-wired RX address %08x "
2109 "overriding with %08x\n",
2110 dev_addr_rx, config_addr);
2111 if (cfg->dir != STEDMA40_PERIPH_TO_MEM)
2112 dev_dbg(d40c->base->dev,
2113 "channel was not configured for peripheral "
2114 "to memory transfer (%d) overriding\n",
2115 cfg->dir);
2116 cfg->dir = STEDMA40_PERIPH_TO_MEM;
2117
2118 config_addr_width = config->src_addr_width;
2119 config_maxburst = config->src_maxburst;
2120
2121 } else if (config->direction == DMA_TO_DEVICE) {
2122 dma_addr_t dev_addr_tx =
2123 d40c->base->plat_data->dev_tx[cfg->dst_dev_type];
2124
2125 config_addr = config->dst_addr;
2126 if (dev_addr_tx)
2127 dev_dbg(d40c->base->dev,
2128 "channel has a pre-wired TX address %08x "
2129 "overriding with %08x\n",
2130 dev_addr_tx, config_addr);
2131 if (cfg->dir != STEDMA40_MEM_TO_PERIPH)
2132 dev_dbg(d40c->base->dev,
2133 "channel was not configured for memory "
2134 "to peripheral transfer (%d) overriding\n",
2135 cfg->dir);
2136 cfg->dir = STEDMA40_MEM_TO_PERIPH;
2137
2138 config_addr_width = config->dst_addr_width;
2139 config_maxburst = config->dst_maxburst;
2140
2141 } else {
2142 dev_err(d40c->base->dev,
2143 "unrecognized channel direction %d\n",
2144 config->direction);
2145 return;
2146 }
2147
2148 switch (config_addr_width) {
2149 case DMA_SLAVE_BUSWIDTH_1_BYTE:
2150 addr_width = STEDMA40_BYTE_WIDTH;
2151 break;
2152 case DMA_SLAVE_BUSWIDTH_2_BYTES:
2153 addr_width = STEDMA40_HALFWORD_WIDTH;
2154 break;
2155 case DMA_SLAVE_BUSWIDTH_4_BYTES:
2156 addr_width = STEDMA40_WORD_WIDTH;
2157 break;
2158 case DMA_SLAVE_BUSWIDTH_8_BYTES:
2159 addr_width = STEDMA40_DOUBLEWORD_WIDTH;
2160 break;
2161 default:
2162 dev_err(d40c->base->dev,
2163 "illegal peripheral address width "
2164 "requested (%d)\n",
2165 config->src_addr_width);
2166 return;
2167 }
2168
2169 if (config_maxburst >= 16)
2170 psize = STEDMA40_PSIZE_LOG_16;
2171 else if (config_maxburst >= 8)
2172 psize = STEDMA40_PSIZE_LOG_8;
2173 else if (config_maxburst >= 4)
2174 psize = STEDMA40_PSIZE_LOG_4;
2175 else
2176 psize = STEDMA40_PSIZE_LOG_1;
2177
2178 /* Set up all the endpoint configs */
2179 cfg->src_info.data_width = addr_width;
2180 cfg->src_info.psize = psize;
2181 cfg->src_info.endianess = STEDMA40_LITTLE_ENDIAN;
2182 cfg->src_info.flow_ctrl = STEDMA40_NO_FLOW_CTRL;
2183 cfg->dst_info.data_width = addr_width;
2184 cfg->dst_info.psize = psize;
2185 cfg->dst_info.endianess = STEDMA40_LITTLE_ENDIAN;
2186 cfg->dst_info.flow_ctrl = STEDMA40_NO_FLOW_CTRL;
2187
2188 /* These settings will take precedence later */
2189 d40c->runtime_addr = config_addr;
2190 d40c->runtime_direction = config->direction;
2191 dev_dbg(d40c->base->dev,
2192 "configured channel %s for %s, data width %d, "
2193 "maxburst %d bytes, LE, no flow control\n",
2194 dma_chan_name(chan),
2195 (config->direction == DMA_FROM_DEVICE) ? "RX" : "TX",
2196 config_addr_width,
2197 config_maxburst);
2198}
2199
2068static int d40_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd, 2200static int d40_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd,
2069 unsigned long arg) 2201 unsigned long arg)
2070{ 2202{
2071 unsigned long flags; 2203 unsigned long flags;
2072 struct d40_chan *d40c = container_of(chan, struct d40_chan, chan); 2204 struct d40_chan *d40c = container_of(chan, struct d40_chan, chan);
2073 2205
2206 if (d40c->phy_chan == NULL) {
2207 dev_err(&d40c->chan.dev->device,
2208 "[%s] Channel is not allocated!\n", __func__);
2209 return -EINVAL;
2210 }
2211
2074 switch (cmd) { 2212 switch (cmd) {
2075 case DMA_TERMINATE_ALL: 2213 case DMA_TERMINATE_ALL:
2076 spin_lock_irqsave(&d40c->lock, flags); 2214 spin_lock_irqsave(&d40c->lock, flags);
@@ -2081,6 +2219,12 @@ static int d40_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd,
2081 return d40_pause(chan); 2219 return d40_pause(chan);
2082 case DMA_RESUME: 2220 case DMA_RESUME:
2083 return d40_resume(chan); 2221 return d40_resume(chan);
2222 case DMA_SLAVE_CONFIG:
2223 d40_set_runtime_config(chan,
2224 (struct dma_slave_config *) arg);
2225 return 0;
2226 default:
2227 break;
2084 } 2228 }
2085 2229
2086 /* Other commands are unimplemented */ 2230 /* Other commands are unimplemented */
@@ -2111,13 +2255,10 @@ static void __init d40_chan_init(struct d40_base *base, struct dma_device *dma,
2111 2255
2112 d40c->log_num = D40_PHY_CHAN; 2256 d40c->log_num = D40_PHY_CHAN;
2113 2257
2114 INIT_LIST_HEAD(&d40c->free);
2115 INIT_LIST_HEAD(&d40c->active); 2258 INIT_LIST_HEAD(&d40c->active);
2116 INIT_LIST_HEAD(&d40c->queue); 2259 INIT_LIST_HEAD(&d40c->queue);
2117 INIT_LIST_HEAD(&d40c->client); 2260 INIT_LIST_HEAD(&d40c->client);
2118 2261
2119 d40c->free_len = 0;
2120
2121 tasklet_init(&d40c->tasklet, dma_tasklet, 2262 tasklet_init(&d40c->tasklet, dma_tasklet,
2122 (unsigned long) d40c); 2263 (unsigned long) d40c);
2123 2264
@@ -2243,6 +2384,14 @@ static int __init d40_phy_res_init(struct d40_base *base)
2243 } 2384 }
2244 spin_lock_init(&base->phy_res[i].lock); 2385 spin_lock_init(&base->phy_res[i].lock);
2245 } 2386 }
2387
2388 /* Mark disabled channels as occupied */
2389 for (i = 0; base->plat_data->disabled_channels[i] != -1; i++) {
2390 base->phy_res[i].allocated_src = D40_ALLOC_PHY;
2391 base->phy_res[i].allocated_dst = D40_ALLOC_PHY;
2392 num_phy_chans_avail--;
2393 }
2394
2246 dev_info(base->dev, "%d of %d physical DMA channels available\n", 2395 dev_info(base->dev, "%d of %d physical DMA channels available\n",
2247 num_phy_chans_avail, base->num_phy_chans); 2396 num_phy_chans_avail, base->num_phy_chans);
2248 2397
@@ -2291,6 +2440,7 @@ static struct d40_base * __init d40_hw_detect_init(struct platform_device *pdev)
2291 int num_log_chans = 0; 2440 int num_log_chans = 0;
2292 int num_phy_chans; 2441 int num_phy_chans;
2293 int i; 2442 int i;
2443 u32 val;
2294 2444
2295 clk = clk_get(&pdev->dev, NULL); 2445 clk = clk_get(&pdev->dev, NULL);
2296 2446
@@ -2329,12 +2479,13 @@ static struct d40_base * __init d40_hw_detect_init(struct platform_device *pdev)
2329 } 2479 }
2330 } 2480 }
2331 2481
2332 i = readl(virtbase + D40_DREG_PERIPHID2); 2482 /* Get silicon revision */
2483 val = readl(virtbase + D40_DREG_PERIPHID2);
2333 2484
2334 if ((i & 0xf) != D40_PERIPHID2_DESIGNER) { 2485 if ((val & 0xf) != D40_PERIPHID2_DESIGNER) {
2335 dev_err(&pdev->dev, 2486 dev_err(&pdev->dev,
2336 "[%s] Unknown designer! Got %x wanted %x\n", 2487 "[%s] Unknown designer! Got %x wanted %x\n",
2337 __func__, i & 0xf, D40_PERIPHID2_DESIGNER); 2488 __func__, val & 0xf, D40_PERIPHID2_DESIGNER);
2338 goto failure; 2489 goto failure;
2339 } 2490 }
2340 2491
@@ -2342,7 +2493,7 @@ static struct d40_base * __init d40_hw_detect_init(struct platform_device *pdev)
2342 num_phy_chans = 4 * (readl(virtbase + D40_DREG_ICFG) & 0x7) + 4; 2493 num_phy_chans = 4 * (readl(virtbase + D40_DREG_ICFG) & 0x7) + 4;
2343 2494
2344 dev_info(&pdev->dev, "hardware revision: %d @ 0x%x\n", 2495 dev_info(&pdev->dev, "hardware revision: %d @ 0x%x\n",
2345 (i >> 4) & 0xf, res->start); 2496 (val >> 4) & 0xf, res->start);
2346 2497
2347 plat_data = pdev->dev.platform_data; 2498 plat_data = pdev->dev.platform_data;
2348 2499
@@ -2364,6 +2515,7 @@ static struct d40_base * __init d40_hw_detect_init(struct platform_device *pdev)
2364 goto failure; 2515 goto failure;
2365 } 2516 }
2366 2517
2518 base->rev = (val >> 4) & 0xf;
2367 base->clk = clk; 2519 base->clk = clk;
2368 base->num_phy_chans = num_phy_chans; 2520 base->num_phy_chans = num_phy_chans;
2369 base->num_log_chans = num_log_chans; 2521 base->num_log_chans = num_log_chans;
@@ -2402,6 +2554,12 @@ static struct d40_base * __init d40_hw_detect_init(struct platform_device *pdev)
2402 if (!base->lcla_pool.alloc_map) 2554 if (!base->lcla_pool.alloc_map)
2403 goto failure; 2555 goto failure;
2404 2556
2557 base->desc_slab = kmem_cache_create(D40_NAME, sizeof(struct d40_desc),
2558 0, SLAB_HWCACHE_ALIGN,
2559 NULL);
2560 if (base->desc_slab == NULL)
2561 goto failure;
2562
2405 return base; 2563 return base;
2406 2564
2407failure: 2565failure:
@@ -2495,6 +2653,78 @@ static void __init d40_hw_init(struct d40_base *base)
2495 2653
2496} 2654}
2497 2655
2656static int __init d40_lcla_allocate(struct d40_base *base)
2657{
2658 unsigned long *page_list;
2659 int i, j;
2660 int ret = 0;
2661
2662 /*
2663 * This is somewhat ugly. We need 8192 bytes that are 18 bit aligned,
2664 * To full fill this hardware requirement without wasting 256 kb
2665 * we allocate pages until we get an aligned one.
2666 */
2667 page_list = kmalloc(sizeof(unsigned long) * MAX_LCLA_ALLOC_ATTEMPTS,
2668 GFP_KERNEL);
2669
2670 if (!page_list) {
2671 ret = -ENOMEM;
2672 goto failure;
2673 }
2674
2675 /* Calculating how many pages that are required */
2676 base->lcla_pool.pages = SZ_1K * base->num_phy_chans / PAGE_SIZE;
2677
2678 for (i = 0; i < MAX_LCLA_ALLOC_ATTEMPTS; i++) {
2679 page_list[i] = __get_free_pages(GFP_KERNEL,
2680 base->lcla_pool.pages);
2681 if (!page_list[i]) {
2682
2683 dev_err(base->dev,
2684 "[%s] Failed to allocate %d pages.\n",
2685 __func__, base->lcla_pool.pages);
2686
2687 for (j = 0; j < i; j++)
2688 free_pages(page_list[j], base->lcla_pool.pages);
2689 goto failure;
2690 }
2691
2692 if ((virt_to_phys((void *)page_list[i]) &
2693 (LCLA_ALIGNMENT - 1)) == 0)
2694 break;
2695 }
2696
2697 for (j = 0; j < i; j++)
2698 free_pages(page_list[j], base->lcla_pool.pages);
2699
2700 if (i < MAX_LCLA_ALLOC_ATTEMPTS) {
2701 base->lcla_pool.base = (void *)page_list[i];
2702 } else {
2703 /* After many attempts, no succees with finding the correct
2704 * alignment try with allocating a big buffer */
2705 dev_warn(base->dev,
2706 "[%s] Failed to get %d pages @ 18 bit align.\n",
2707 __func__, base->lcla_pool.pages);
2708 base->lcla_pool.base_unaligned = kmalloc(SZ_1K *
2709 base->num_phy_chans +
2710 LCLA_ALIGNMENT,
2711 GFP_KERNEL);
2712 if (!base->lcla_pool.base_unaligned) {
2713 ret = -ENOMEM;
2714 goto failure;
2715 }
2716
2717 base->lcla_pool.base = PTR_ALIGN(base->lcla_pool.base_unaligned,
2718 LCLA_ALIGNMENT);
2719 }
2720
2721 writel(virt_to_phys(base->lcla_pool.base),
2722 base->virtbase + D40_DREG_LCLA);
2723failure:
2724 kfree(page_list);
2725 return ret;
2726}
2727
2498static int __init d40_probe(struct platform_device *pdev) 2728static int __init d40_probe(struct platform_device *pdev)
2499{ 2729{
2500 int err; 2730 int err;
@@ -2554,41 +2784,11 @@ static int __init d40_probe(struct platform_device *pdev)
2554 __func__); 2784 __func__);
2555 goto failure; 2785 goto failure;
2556 } 2786 }
2557 /* Get IO for logical channel link address */
2558 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "lcla");
2559 if (!res) {
2560 ret = -ENOENT;
2561 dev_err(&pdev->dev,
2562 "[%s] No \"lcla\" resource defined\n",
2563 __func__);
2564 goto failure;
2565 }
2566 2787
2567 base->lcla_pool.base_size = resource_size(res); 2788 ret = d40_lcla_allocate(base);
2568 base->lcla_pool.phy = res->start; 2789 if (ret) {
2569 2790 dev_err(&pdev->dev, "[%s] Failed to allocate LCLA area\n",
2570 if (request_mem_region(res->start, resource_size(res), 2791 __func__);
2571 D40_NAME " I/O lcla") == NULL) {
2572 ret = -EBUSY;
2573 dev_err(&pdev->dev,
2574 "[%s] Failed to request LCLA region 0x%x-0x%x\n",
2575 __func__, res->start, res->end);
2576 goto failure;
2577 }
2578 val = readl(base->virtbase + D40_DREG_LCLA);
2579 if (res->start != val && val != 0) {
2580 dev_warn(&pdev->dev,
2581 "[%s] Mismatch LCLA dma 0x%x, def 0x%x\n",
2582 __func__, val, res->start);
2583 } else
2584 writel(res->start, base->virtbase + D40_DREG_LCLA);
2585
2586 base->lcla_pool.base = ioremap(res->start, resource_size(res));
2587 if (!base->lcla_pool.base) {
2588 ret = -ENOMEM;
2589 dev_err(&pdev->dev,
2590 "[%s] Failed to ioremap LCLA 0x%x-0x%x\n",
2591 __func__, res->start, res->end);
2592 goto failure; 2792 goto failure;
2593 } 2793 }
2594 2794
@@ -2616,11 +2816,15 @@ static int __init d40_probe(struct platform_device *pdev)
2616 2816
2617failure: 2817failure:
2618 if (base) { 2818 if (base) {
2819 if (base->desc_slab)
2820 kmem_cache_destroy(base->desc_slab);
2619 if (base->virtbase) 2821 if (base->virtbase)
2620 iounmap(base->virtbase); 2822 iounmap(base->virtbase);
2621 if (base->lcla_pool.phy) 2823 if (!base->lcla_pool.base_unaligned && base->lcla_pool.base)
2622 release_mem_region(base->lcla_pool.phy, 2824 free_pages((unsigned long)base->lcla_pool.base,
2623 base->lcla_pool.base_size); 2825 base->lcla_pool.pages);
2826 if (base->lcla_pool.base_unaligned)
2827 kfree(base->lcla_pool.base_unaligned);
2624 if (base->phy_lcpa) 2828 if (base->phy_lcpa)
2625 release_mem_region(base->phy_lcpa, 2829 release_mem_region(base->phy_lcpa,
2626 base->lcpa_size); 2830 base->lcpa_size);
diff --git a/drivers/dma/ste_dma40_ll.c b/drivers/dma/ste_dma40_ll.c
index 561fdd8a80c1..d937f76d6e2e 100644
--- a/drivers/dma/ste_dma40_ll.c
+++ b/drivers/dma/ste_dma40_ll.c
@@ -315,11 +315,8 @@ int d40_log_sg_to_dev(struct d40_lcla_elem *lcla,
315 int total_size = 0; 315 int total_size = 0;
316 struct scatterlist *current_sg = sg; 316 struct scatterlist *current_sg = sg;
317 int i; 317 int i;
318 u32 next_lli_off_dst; 318 u32 next_lli_off_dst = 0;
319 u32 next_lli_off_src; 319 u32 next_lli_off_src = 0;
320
321 next_lli_off_src = 0;
322 next_lli_off_dst = 0;
323 320
324 for_each_sg(sg, current_sg, sg_len, i) { 321 for_each_sg(sg, current_sg, sg_len, i) {
325 total_size += sg_dma_len(current_sg); 322 total_size += sg_dma_len(current_sg);
@@ -351,7 +348,7 @@ int d40_log_sg_to_dev(struct d40_lcla_elem *lcla,
351 sg_dma_len(current_sg), 348 sg_dma_len(current_sg),
352 next_lli_off_src, 349 next_lli_off_src,
353 lcsp->lcsp1, src_data_width, 350 lcsp->lcsp1, src_data_width,
354 term_int && !next_lli_off_src, 351 false,
355 true); 352 true);
356 d40_log_fill_lli(&lli->dst[i], 353 d40_log_fill_lli(&lli->dst[i],
357 dev_addr, 354 dev_addr,
@@ -375,7 +372,7 @@ int d40_log_sg_to_dev(struct d40_lcla_elem *lcla,
375 sg_dma_len(current_sg), 372 sg_dma_len(current_sg),
376 next_lli_off_src, 373 next_lli_off_src,
377 lcsp->lcsp1, src_data_width, 374 lcsp->lcsp1, src_data_width,
378 term_int && !next_lli_off_src, 375 false,
379 false); 376 false);
380 } 377 }
381 } 378 }
@@ -423,32 +420,35 @@ int d40_log_sg_to_lli(int lcla_id,
423 return total_size; 420 return total_size;
424} 421}
425 422
426void d40_log_lli_write(struct d40_log_lli_full *lcpa, 423int d40_log_lli_write(struct d40_log_lli_full *lcpa,
427 struct d40_log_lli *lcla_src, 424 struct d40_log_lli *lcla_src,
428 struct d40_log_lli *lcla_dst, 425 struct d40_log_lli *lcla_dst,
429 struct d40_log_lli *lli_dst, 426 struct d40_log_lli *lli_dst,
430 struct d40_log_lli *lli_src, 427 struct d40_log_lli *lli_src,
431 int llis_per_log) 428 int llis_per_log)
432{ 429{
433 u32 slos = 0; 430 u32 slos;
434 u32 dlos = 0; 431 u32 dlos;
435 int i; 432 int i;
436 433
437 lcpa->lcsp0 = lli_src->lcsp02; 434 writel(lli_src->lcsp02, &lcpa->lcsp0);
438 lcpa->lcsp1 = lli_src->lcsp13; 435 writel(lli_src->lcsp13, &lcpa->lcsp1);
439 lcpa->lcsp2 = lli_dst->lcsp02; 436 writel(lli_dst->lcsp02, &lcpa->lcsp2);
440 lcpa->lcsp3 = lli_dst->lcsp13; 437 writel(lli_dst->lcsp13, &lcpa->lcsp3);
441 438
442 slos = lli_src->lcsp13 & D40_MEM_LCSP1_SLOS_MASK; 439 slos = lli_src->lcsp13 & D40_MEM_LCSP1_SLOS_MASK;
443 dlos = lli_dst->lcsp13 & D40_MEM_LCSP3_DLOS_MASK; 440 dlos = lli_dst->lcsp13 & D40_MEM_LCSP3_DLOS_MASK;
444 441
445 for (i = 0; (i < llis_per_log) && slos && dlos; i++) { 442 for (i = 0; (i < llis_per_log) && slos && dlos; i++) {
446 writel(lli_src[i+1].lcsp02, &lcla_src[i].lcsp02); 443 writel(lli_src[i + 1].lcsp02, &lcla_src[i].lcsp02);
447 writel(lli_src[i+1].lcsp13, &lcla_src[i].lcsp13); 444 writel(lli_src[i + 1].lcsp13, &lcla_src[i].lcsp13);
448 writel(lli_dst[i+1].lcsp02, &lcla_dst[i].lcsp02); 445 writel(lli_dst[i + 1].lcsp02, &lcla_dst[i].lcsp02);
449 writel(lli_dst[i+1].lcsp13, &lcla_dst[i].lcsp13); 446 writel(lli_dst[i + 1].lcsp13, &lcla_dst[i].lcsp13);
450 447
451 slos = lli_src[i+1].lcsp13 & D40_MEM_LCSP1_SLOS_MASK; 448 slos = lli_src[i + 1].lcsp13 & D40_MEM_LCSP1_SLOS_MASK;
452 dlos = lli_dst[i+1].lcsp13 & D40_MEM_LCSP3_DLOS_MASK; 449 dlos = lli_dst[i + 1].lcsp13 & D40_MEM_LCSP3_DLOS_MASK;
453 } 450 }
451
452 return i;
453
454} 454}
diff --git a/drivers/dma/ste_dma40_ll.h b/drivers/dma/ste_dma40_ll.h
index 2029280cb332..9c0fa2f5fe57 100644
--- a/drivers/dma/ste_dma40_ll.h
+++ b/drivers/dma/ste_dma40_ll.h
@@ -13,6 +13,9 @@
13#define D40_DREG_PCDELTA (8 * 4) 13#define D40_DREG_PCDELTA (8 * 4)
14#define D40_LLI_ALIGN 16 /* LLI alignment must be 16 bytes. */ 14#define D40_LLI_ALIGN 16 /* LLI alignment must be 16 bytes. */
15 15
16#define D40_LCPA_CHAN_SIZE 32
17#define D40_LCPA_CHAN_DST_DELTA 16
18
16#define D40_TYPE_TO_GROUP(type) (type / 16) 19#define D40_TYPE_TO_GROUP(type) (type / 16)
17#define D40_TYPE_TO_EVENT(type) (type % 16) 20#define D40_TYPE_TO_EVENT(type) (type % 16)
18 21
@@ -336,12 +339,12 @@ int d40_log_sg_to_dev(struct d40_lcla_elem *lcla,
336 bool term_int, dma_addr_t dev_addr, int max_len, 339 bool term_int, dma_addr_t dev_addr, int max_len,
337 int llis_per_log); 340 int llis_per_log);
338 341
339void d40_log_lli_write(struct d40_log_lli_full *lcpa, 342int d40_log_lli_write(struct d40_log_lli_full *lcpa,
340 struct d40_log_lli *lcla_src, 343 struct d40_log_lli *lcla_src,
341 struct d40_log_lli *lcla_dst, 344 struct d40_log_lli *lcla_dst,
342 struct d40_log_lli *lli_dst, 345 struct d40_log_lli *lli_dst,
343 struct d40_log_lli *lli_src, 346 struct d40_log_lli *lli_src,
344 int llis_per_log); 347 int llis_per_log);
345 348
346int d40_log_sg_to_lli(int lcla_id, 349int d40_log_sg_to_lli(int lcla_id,
347 struct scatterlist *sg, 350 struct scatterlist *sg,
diff --git a/drivers/dma/timb_dma.c b/drivers/dma/timb_dma.c
index a1bf77c1993f..2ec1ed56f204 100644
--- a/drivers/dma/timb_dma.c
+++ b/drivers/dma/timb_dma.c
@@ -200,8 +200,8 @@ static int td_fill_desc(struct timb_dma_chan *td_chan, u8 *dma_desc,
200 return -EINVAL; 200 return -EINVAL;
201 } 201 }
202 202
203 dev_dbg(chan2dev(&td_chan->chan), "desc: %p, addr: %p\n", 203 dev_dbg(chan2dev(&td_chan->chan), "desc: %p, addr: 0x%llx\n",
204 dma_desc, (void *)sg_dma_address(sg)); 204 dma_desc, (unsigned long long)sg_dma_address(sg));
205 205
206 dma_desc[7] = (sg_dma_address(sg) >> 24) & 0xff; 206 dma_desc[7] = (sg_dma_address(sg) >> 24) & 0xff;
207 dma_desc[6] = (sg_dma_address(sg) >> 16) & 0xff; 207 dma_desc[6] = (sg_dma_address(sg) >> 16) & 0xff;
@@ -382,7 +382,7 @@ static struct timb_dma_desc *td_alloc_init_desc(struct timb_dma_chan *td_chan)
382 td_desc = kzalloc(sizeof(struct timb_dma_desc), GFP_KERNEL); 382 td_desc = kzalloc(sizeof(struct timb_dma_desc), GFP_KERNEL);
383 if (!td_desc) { 383 if (!td_desc) {
384 dev_err(chan2dev(chan), "Failed to alloc descriptor\n"); 384 dev_err(chan2dev(chan), "Failed to alloc descriptor\n");
385 goto err; 385 goto out;
386 } 386 }
387 387
388 td_desc->desc_list_len = td_chan->desc_elems * TIMB_DMA_DESC_SIZE; 388 td_desc->desc_list_len = td_chan->desc_elems * TIMB_DMA_DESC_SIZE;
@@ -410,7 +410,7 @@ static struct timb_dma_desc *td_alloc_init_desc(struct timb_dma_chan *td_chan)
410err: 410err:
411 kfree(td_desc->desc_list); 411 kfree(td_desc->desc_list);
412 kfree(td_desc); 412 kfree(td_desc);
413 413out:
414 return NULL; 414 return NULL;
415 415
416} 416}
diff --git a/drivers/pci/intel-iommu.c b/drivers/pci/intel-iommu.c
index 6a5af18faf68..b0de57947189 100644
--- a/drivers/pci/intel-iommu.c
+++ b/drivers/pci/intel-iommu.c
@@ -3030,6 +3030,34 @@ static void __init iommu_exit_mempool(void)
3030 3030
3031} 3031}
3032 3032
3033static void quirk_ioat_snb_local_iommu(struct pci_dev *pdev)
3034{
3035 struct dmar_drhd_unit *drhd;
3036 u32 vtbar;
3037 int rc;
3038
3039 /* We know that this device on this chipset has its own IOMMU.
3040 * If we find it under a different IOMMU, then the BIOS is lying
3041 * to us. Hope that the IOMMU for this device is actually
3042 * disabled, and it needs no translation...
3043 */
3044 rc = pci_bus_read_config_dword(pdev->bus, PCI_DEVFN(0, 0), 0xb0, &vtbar);
3045 if (rc) {
3046 /* "can't" happen */
3047 dev_info(&pdev->dev, "failed to run vt-d quirk\n");
3048 return;
3049 }
3050 vtbar &= 0xffff0000;
3051
3052 /* we know that the this iommu should be at offset 0xa000 from vtbar */
3053 drhd = dmar_find_matched_drhd_unit(pdev);
3054 if (WARN_TAINT_ONCE(!drhd || drhd->reg_base_addr - vtbar != 0xa000,
3055 TAINT_FIRMWARE_WORKAROUND,
3056 "BIOS assigned incorrect VT-d unit for Intel(R) QuickData Technology device\n"))
3057 pdev->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
3058}
3059DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB, quirk_ioat_snb_local_iommu);
3060
3033static void __init init_no_remapping_devices(void) 3061static void __init init_no_remapping_devices(void)
3034{ 3062{
3035 struct dmar_drhd_unit *drhd; 3063 struct dmar_drhd_unit *drhd;
generated by cgit v1.2.2 (git 2.25.0) at 2024-11-27 08:42:55 -0500